import copy
import logging
from functools import partial
from typing import Any, Dict, List, Optional
import numpy as np
# Use cloudpickle instead of pickle to make lambda funcs in HyperOpt pickleable
from ray import cloudpickle
from ray.tune.error import TuneError
from ray.tune.result import DEFAULT_METRIC
from ray.tune.search import (
UNDEFINED_METRIC_MODE,
UNDEFINED_SEARCH_SPACE,
UNRESOLVED_SEARCH_SPACE,
Searcher,
)
from ray.tune.search.sample import (
Categorical,
Domain,
Float,
Integer,
LogUniform,
Normal,
Quantized,
Uniform,
)
from ray.tune.search.variant_generator import assign_value, parse_spec_vars
from ray.tune.utils import flatten_dict
try:
hyperopt_logger = logging.getLogger("hyperopt")
hyperopt_logger.setLevel(logging.WARNING)
import hyperopt as hpo
from hyperopt.pyll import Apply
except ImportError:
hpo = None
Apply = None
logger = logging.getLogger(__name__)
HYPEROPT_UNDEFINED_DETAILS = (
" This issue can also come up with HyperOpt if your search space only "
"contains constant variables, which is not supported by HyperOpt. In that case, "
"don't pass any searcher or add sample variables to the search space."
)
[docs]
class HyperOptSearch(Searcher):
"""A wrapper around HyperOpt to provide trial suggestions.
HyperOpt a Python library for serial and parallel optimization
over awkward search spaces, which may include real-valued, discrete,
and conditional dimensions. More info can be found at
http://hyperopt.github.io/hyperopt.
HyperOptSearch uses the Tree-structured Parzen Estimators algorithm,
though it can be trivially extended to support any algorithm HyperOpt
supports.
To use this search algorithm, you will need to install HyperOpt:
.. code-block:: bash
pip install -U hyperopt
Parameters:
space: HyperOpt configuration. Parameters will be sampled
from this configuration and will be used to override
parameters generated in the variant generation process.
metric: The training result objective value attribute. If None
but a mode was passed, the anonymous metric `_metric` will be used
per default.
mode: One of {min, max}. Determines whether objective is
minimizing or maximizing the metric attribute.
points_to_evaluate: Initial parameter suggestions to be run
first. This is for when you already have some good parameters
you want to run first to help the algorithm make better suggestions
for future parameters. Needs to be a list of dicts containing the
configurations.
n_initial_points: number of random evaluations of the
objective function before starting to aproximate it with
tree parzen estimators. Defaults to 20.
random_state_seed: seed for reproducible
results. Defaults to None.
gamma: parameter governing the tree parzen
estimators suggestion algorithm. Defaults to 0.25.
Tune automatically converts search spaces to HyperOpt's format:
.. code-block:: python
config = {
'width': tune.uniform(0, 20),
'height': tune.uniform(-100, 100),
'activation': tune.choice(["relu", "tanh"])
}
current_best_params = [{
'width': 10,
'height': 0,
'activation': "relu",
}]
hyperopt_search = HyperOptSearch(
metric="mean_loss", mode="min",
points_to_evaluate=current_best_params)
tuner = tune.Tuner(
trainable,
tune_config=tune.TuneConfig(
search_alg=hyperopt_search
),
param_space=config
)
tuner.fit()
If you would like to pass the search space manually, the code would
look like this:
.. code-block:: python
space = {
'width': hp.uniform('width', 0, 20),
'height': hp.uniform('height', -100, 100),
'activation': hp.choice("activation", ["relu", "tanh"])
}
current_best_params = [{
'width': 10,
'height': 0,
'activation': "relu",
}]
hyperopt_search = HyperOptSearch(
space, metric="mean_loss", mode="min",
points_to_evaluate=current_best_params)
tuner = tune.Tuner(
trainable,
tune_config=tune.TuneConfig(
search_alg=hyperopt_search
),
)
tuner.fit()
"""
def __init__(
self,
space: Optional[Dict] = None,
metric: Optional[str] = None,
mode: Optional[str] = None,
points_to_evaluate: Optional[List[Dict]] = None,
n_initial_points: int = 20,
random_state_seed: Optional[int] = None,
gamma: float = 0.25,
):
assert (
hpo is not None
), "HyperOpt must be installed! Run `pip install hyperopt`."
if mode:
assert mode in ["min", "max"], "`mode` must be 'min' or 'max'."
super(HyperOptSearch, self).__init__(
metric=metric,
mode=mode,
)
# hyperopt internally minimizes, so "max" => -1
if mode == "max":
self.metric_op = -1.0
elif mode == "min":
self.metric_op = 1.0
if n_initial_points is None:
self.algo = hpo.tpe.suggest
else:
self.algo = partial(hpo.tpe.suggest, n_startup_jobs=n_initial_points)
if gamma is not None:
self.algo = partial(self.algo, gamma=gamma)
self._points_to_evaluate = copy.deepcopy(points_to_evaluate)
self._live_trial_mapping = {}
self.rstate = np.random.RandomState(random_state_seed)
self.domain = None
if isinstance(space, dict) and space:
resolved_vars, domain_vars, grid_vars = parse_spec_vars(space)
if domain_vars or grid_vars:
logger.warning(
UNRESOLVED_SEARCH_SPACE.format(par="space", cls=type(self))
)
space = self.convert_search_space(space)
self._space = space
self._setup_hyperopt()
def _setup_hyperopt(self) -> None:
from hyperopt.fmin import generate_trials_to_calculate
if not self._space:
raise RuntimeError(
UNDEFINED_SEARCH_SPACE.format(
cls=self.__class__.__name__, space="space"
)
+ HYPEROPT_UNDEFINED_DETAILS
)
if self._metric is None and self._mode:
# If only a mode was passed, use anonymous metric
self._metric = DEFAULT_METRIC
if self._points_to_evaluate is None:
self._hpopt_trials = hpo.Trials()
self._points_to_evaluate = 0
else:
assert isinstance(self._points_to_evaluate, (list, tuple))
for i in range(len(self._points_to_evaluate)):
config = self._points_to_evaluate[i]
self._convert_categories_to_indices(config)
# HyperOpt treats initial points as LIFO, reverse to get FIFO
self._points_to_evaluate = list(reversed(self._points_to_evaluate))
self._hpopt_trials = generate_trials_to_calculate(self._points_to_evaluate)
self._hpopt_trials.refresh()
self._points_to_evaluate = len(self._points_to_evaluate)
self.domain = hpo.Domain(lambda spc: spc, self._space)
def _convert_categories_to_indices(self, config) -> None:
"""Convert config parameters for categories into hyperopt-compatible
representations where instead the index of the category is expected."""
def _lookup(config_dict, space_dict, key):
if isinstance(config_dict[key], dict):
for k in config_dict[key]:
_lookup(config_dict[key], space_dict[key], k)
else:
if (
key in space_dict
and isinstance(space_dict[key], hpo.base.pyll.Apply)
and space_dict[key].name == "switch"
):
if len(space_dict[key].pos_args) > 0:
categories = [
a.obj
for a in space_dict[key].pos_args[1:]
if a.name == "literal"
]
try:
idx = categories.index(config_dict[key])
except ValueError as exc:
msg = (
f"Did not find category with value "
f"`{config_dict[key]}` in "
f"hyperopt parameter `{key}`. "
)
if isinstance(config_dict[key], int):
msg += (
"In previous versions, a numerical "
"index was expected for categorical "
"values of `points_to_evaluate`, "
"but in ray>=1.2.0, the categorical "
"value is expected to be directly "
"provided. "
)
msg += "Please make sure the specified category is valid."
raise ValueError(msg) from exc
config_dict[key] = idx
for k in config:
_lookup(config, self._space, k)
def set_search_properties(
self, metric: Optional[str], mode: Optional[str], config: Dict, **spec
) -> bool:
if self.domain:
return False
space = self.convert_search_space(config)
self._space = space
if metric:
self._metric = metric
if mode:
self._mode = mode
self.metric_op = -1.0 if self._mode == "max" else 1.0
self._setup_hyperopt()
return True
def suggest(self, trial_id: str) -> Optional[Dict]:
if not self.domain:
raise RuntimeError(
UNDEFINED_SEARCH_SPACE.format(
cls=self.__class__.__name__, space="space"
)
+ HYPEROPT_UNDEFINED_DETAILS
)
if not self._metric or not self._mode:
raise RuntimeError(
UNDEFINED_METRIC_MODE.format(
cls=self.__class__.__name__, metric=self._metric, mode=self._mode
)
)
if self._points_to_evaluate > 0:
using_point_to_evaluate = True
new_trial = self._hpopt_trials.trials[self._points_to_evaluate - 1]
self._points_to_evaluate -= 1
else:
using_point_to_evaluate = False
new_ids = self._hpopt_trials.new_trial_ids(1)
self._hpopt_trials.refresh()
# Get new suggestion from Hyperopt
new_trials = self.algo(
new_ids,
self.domain,
self._hpopt_trials,
self.rstate.randint(2**31 - 1),
)
self._hpopt_trials.insert_trial_docs(new_trials)
self._hpopt_trials.refresh()
new_trial = new_trials[0]
self._live_trial_mapping[trial_id] = (new_trial["tid"], new_trial)
# Taken from HyperOpt.base.evaluate
config = hpo.base.spec_from_misc(new_trial["misc"])
# We have to flatten nested spaces here so parameter names match
config = flatten_dict(config, flatten_list=True)
ctrl = hpo.base.Ctrl(self._hpopt_trials, current_trial=new_trial)
memo = self.domain.memo_from_config(config)
hpo.utils.use_obj_for_literal_in_memo(
self.domain.expr, ctrl, hpo.base.Ctrl, memo
)
try:
suggested_config = hpo.pyll.rec_eval(
self.domain.expr,
memo=memo,
print_node_on_error=self.domain.rec_eval_print_node_on_error,
)
except (AssertionError, TypeError) as e:
if using_point_to_evaluate and (
isinstance(e, AssertionError) or "GarbageCollected" in str(e)
):
raise ValueError(
"HyperOpt encountered a GarbageCollected switch argument. "
"Usually this is caused by a config in "
"`points_to_evaluate` "
"missing a key present in `space`. Ensure that "
"`points_to_evaluate` contains "
"all non-constant keys from `space`.\n"
"Config from `points_to_evaluate`: "
f"{config}\n"
"HyperOpt search space: "
f"{self._space}"
) from e
raise e
return copy.deepcopy(suggested_config)
def on_trial_result(self, trial_id: str, result: Dict) -> None:
ho_trial = self._get_hyperopt_trial(trial_id)
if ho_trial is None:
return
now = hpo.utils.coarse_utcnow()
ho_trial["book_time"] = now
ho_trial["refresh_time"] = now
[docs]
def on_trial_complete(
self, trial_id: str, result: Optional[Dict] = None, error: bool = False
) -> None:
"""Notification for the completion of trial.
The result is internally negated when interacting with HyperOpt
so that HyperOpt can "maximize" this value, as it minimizes on default.
"""
ho_trial = self._get_hyperopt_trial(trial_id)
if ho_trial is None:
return
ho_trial["refresh_time"] = hpo.utils.coarse_utcnow()
if error:
ho_trial["state"] = hpo.base.JOB_STATE_ERROR
ho_trial["misc"]["error"] = (str(TuneError), "Tune Error")
self._hpopt_trials.refresh()
elif result:
self._process_result(trial_id, result)
del self._live_trial_mapping[trial_id]
def _process_result(self, trial_id: str, result: Dict) -> None:
ho_trial = self._get_hyperopt_trial(trial_id)
if not ho_trial:
return
ho_trial["refresh_time"] = hpo.utils.coarse_utcnow()
ho_trial["state"] = hpo.base.JOB_STATE_DONE
hp_result = self._to_hyperopt_result(result)
ho_trial["result"] = hp_result
self._hpopt_trials.refresh()
def _to_hyperopt_result(self, result: Dict) -> Dict:
try:
return {"loss": self.metric_op * result[self.metric], "status": "ok"}
except KeyError as e:
raise RuntimeError(
f"Hyperopt expected to see the metric `{self.metric}` in the "
f"last result, but it was not found. To fix this, make "
f"sure your call to `tune.report` or your return value of "
f"your trainable class `step()` contains the above metric "
f"as a key."
) from e
def _get_hyperopt_trial(self, trial_id: str) -> Optional[Dict]:
if trial_id not in self._live_trial_mapping:
return
hyperopt_tid = self._live_trial_mapping[trial_id][0]
return [t for t in self._hpopt_trials.trials if t["tid"] == hyperopt_tid][0]
def get_state(self) -> Dict:
return {
"hyperopt_trials": self._hpopt_trials,
"rstate": self.rstate.get_state(),
}
def set_state(self, state: Dict) -> None:
self._hpopt_trials = state["hyperopt_trials"]
self.rstate.set_state(state["rstate"])
def save(self, checkpoint_path: str) -> None:
save_object = self.__dict__.copy()
save_object["__rstate"] = self.rstate.get_state()
with open(checkpoint_path, "wb") as f:
cloudpickle.dump(save_object, f)
def restore(self, checkpoint_path: str) -> None:
with open(checkpoint_path, "rb") as f:
save_object = cloudpickle.load(f)
if "__rstate" not in save_object:
# Backwards compatibility
self.set_state(save_object)
else:
self.rstate.set_state(save_object.pop("__rstate"))
self.__dict__.update(save_object)
@staticmethod
def convert_search_space(spec: Dict, prefix: str = "") -> Dict:
spec = copy.deepcopy(spec)
resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
if not domain_vars and not grid_vars:
return {}
if grid_vars:
raise ValueError(
"Grid search parameters cannot be automatically converted "
"to a HyperOpt search space."
)
def resolve_value(par: str, domain: Domain) -> Any:
quantize = None
sampler = domain.get_sampler()
if isinstance(sampler, Quantized):
quantize = sampler.q
sampler = sampler.sampler
if isinstance(domain, Float):
if isinstance(sampler, LogUniform):
if quantize:
return hpo.hp.qloguniform(
par, np.log(domain.lower), np.log(domain.upper), quantize
)
return hpo.hp.loguniform(
par, np.log(domain.lower), np.log(domain.upper)
)
elif isinstance(sampler, Uniform):
if quantize:
return hpo.hp.quniform(
par, domain.lower, domain.upper, quantize
)
return hpo.hp.uniform(par, domain.lower, domain.upper)
elif isinstance(sampler, Normal):
if quantize:
return hpo.hp.qnormal(par, sampler.mean, sampler.sd, quantize)
return hpo.hp.normal(par, sampler.mean, sampler.sd)
elif isinstance(domain, Integer):
if isinstance(sampler, LogUniform):
if quantize:
return hpo.base.pyll.scope.int(
hpo.hp.qloguniform(
par,
np.log(domain.lower),
np.log(domain.upper),
quantize,
)
)
return hpo.base.pyll.scope.int(
hpo.hp.qloguniform(
par, np.log(domain.lower), np.log(domain.upper - 1), 1.0
)
)
elif isinstance(sampler, Uniform):
if quantize:
return hpo.base.pyll.scope.int(
hpo.hp.quniform(
par, domain.lower, domain.upper - 1, quantize
)
)
return hpo.hp.uniformint(par, domain.lower, high=domain.upper - 1)
elif isinstance(domain, Categorical):
if isinstance(sampler, Uniform):
return hpo.hp.choice(
par,
[
(
HyperOptSearch.convert_search_space(
category, prefix=par
)
if isinstance(category, dict)
else (
HyperOptSearch.convert_search_space(
dict(enumerate(category)), prefix=f"{par}/{i}"
)
if isinstance(category, list)
and len(category) > 0
and isinstance(category[0], Domain)
else (
resolve_value(f"{par}/{i}", category)
if isinstance(category, Domain)
else category
)
)
)
for i, category in enumerate(domain.categories)
],
)
raise ValueError(
"HyperOpt does not support parameters of type "
"`{}` with samplers of type `{}`".format(
type(domain).__name__, type(domain.sampler).__name__
)
)
for path, domain in domain_vars:
par = "/".join([str(p) for p in ((prefix,) + path if prefix else path)])
value = resolve_value(par, domain)
assign_value(spec, path, value)
return spec