from typing import Optional, Type, Union
from ray.rllib.algorithms.algorithm_config import AlgorithmConfig, NotProvided
from ray.rllib.algorithms.marwil.marwil import MARWIL, MARWILConfig
from ray.rllib.connectors.common.add_observations_from_episodes_to_batch import (
AddObservationsFromEpisodesToBatch,
)
from ray.rllib.connectors.learner.add_next_observations_from_episodes_to_train_batch import ( # noqa
AddNextObservationsFromEpisodesToTrainBatch,
)
from ray.rllib.core.learner.learner import Learner
from ray.rllib.core.rl_module.rl_module import RLModuleSpec
from ray.rllib.utils.annotations import override
from ray.rllib.utils.typing import LearningRateOrSchedule, RLModuleSpecType
[docs]
class IQLConfig(MARWILConfig):
"""Defines a configuration class from which a new IQL Algorithm can be built
Examples:
>>> from pathlib import Path # doctest: +SKIP_EXAMPLE
>>> import gymnasium as gym
>>> from ray.rllib.algorithms.iql import IQLConfig
>>> # Get the path to the Pendulum data in the rllib folder.
>>> base_path = Path(__file__).parents[2]
>>> data_path = base_path / "offline/tests/data/pendulum/pendulum-v1_enormous"
>>> config = IQLConfig().training(actor_lr=0.00001, gamma=0.99)
>>> # Set the observation and action spaces of the data.
>>> env = gym.make("Pendulum-v1")
>>> config = config.environment(
... observation_space=env.observation_space,
... action_space=env.action_space,
... )
>>> config = config.offline_data(
... input_=[data_path.as_posix()],
... # Run exactly one update per training iteration.
... dataset_num_iters_per_learner=1,
... )
>>> # Build an Algorithm object from the config and run 1 training iteration.
>>> algo = config.build()
>>> _ = algo.train()
>>> from ray.rllib.algorithms.iql import IQLConfig
>>> from ray import tune
>>> config = IQLConfig()
>>> # Print out some default values.
>>> print(config.beta)
0.1
>>> # Update the config object.
>>> _ = config.training(
... lr=tune.grid_search([0.001, 0.0001]), beta=0.75
... )
>>> # Set the config object's data path.
>>> _ = config.offline_data(
... input_=[data_path.as_posix()],
... dataset_num_iters_per_learner=1,
... )
>>> # Set the observation and action spaces of the data.
>>> _ = config.environment(
... observation_space=env.observation_space,
... action_space=env.action_space,
... )
>>> # Use to_dict() to get the old-style python config dict
>>> # when running with tune.
>>> print("[ignore]"); results = tune.Tuner(
... "IQL",
... param_space=config.to_dict(),
... run_config=tune.RunConfig(stop={"training_iteration": 1}),
... ).fit() # doctest: +ELLIPSIS
[ignore]...
>>> results.num_errors
0
"""
def __init__(self, algo_class=None):
"""Initializes a IQLConfig instance."""
super().__init__(algo_class=algo_class or IQL)
# fmt: off
# __sphinx_doc_begin__
# The temperature for the actor loss.
self.beta = 0.1
# The expectile to use in expectile regression.
self.expectile = 0.8
# The learning rates for the actor, critic and value network(s).
self.actor_lr = 3e-4
self.critic_lr = 3e-4
self.value_lr = 3e-4
# Set `lr` parameter to `None` and ensure it is not used.
self.lr = None
# If a twin-Q architecture should be used (advisable).
self.twin_q = True
# How often the target network should be updated.
self.target_network_update_freq = 0
# The weight for Polyak averaging.
self.tau = 1.0
# __sphinx_doc_end__
# fmt: on
[docs]
@override(MARWILConfig)
def training(
self,
*,
twin_q: Optional[bool] = NotProvided,
expectile: Optional[float] = NotProvided,
actor_lr: Optional[LearningRateOrSchedule] = NotProvided,
critic_lr: Optional[LearningRateOrSchedule] = NotProvided,
value_lr: Optional[LearningRateOrSchedule] = NotProvided,
target_network_update_freq: Optional[int] = NotProvided,
tau: Optional[float] = NotProvided,
**kwargs,
) -> "IQLConfig":
"""Sets the training related configuration.
Args:
twin_q: If a twin-Q architecture should be used (advisable).
expectile: The expectile to use in expectile regression for the value
function. For high expectiles the value function tries to match
the upper tail of the Q-value distribution.
actor_lr: The learning rate for the actor network. Actor learning rates
greater than critic learning rates work well in experiments.
critic_lr: The learning rate for the Q-network. Critic learning rates
greater than value function learning rates work well in experiments.
value_lr: The learning rate for the value function network.
target_network_update_freq: The number of timesteps in between the target
Q-network is fixed. Note, too high values here could harm convergence.
The target network is updated via Polyak-averaging.
tau: The update parameter for Polyak-averaging of the target Q-network.
The higher this value the faster the weights move towards the actual
Q-network.
**kwargs: Additional config settings, forwarded to the parent
`MARWILConfig.training()` method.
Returns:
This updated `AlgorithmConfig` object.
"""
super().training(**kwargs)
if twin_q is not NotProvided:
self.twin_q = twin_q
if expectile is not NotProvided:
self.expectile = expectile
if actor_lr is not NotProvided:
self.actor_lr = actor_lr
if critic_lr is not NotProvided:
self.critic_lr = critic_lr
if value_lr is not NotProvided:
self.value_lr = value_lr
if target_network_update_freq is not NotProvided:
self.target_network_update_freq = target_network_update_freq
if tau is not NotProvided:
self.tau = tau
return self
[docs]
@override(MARWILConfig)
def get_default_learner_class(self) -> Union[Type["Learner"], str]:
if self.framework_str == "torch":
from ray.rllib.algorithms.iql.torch.iql_torch_learner import IQLTorchLearner
return IQLTorchLearner
else:
raise ValueError(
f"The framework {self.framework_str} is not supported. "
"Use `'torch'` instead."
)
[docs]
@override(MARWILConfig)
def get_default_rl_module_spec(self) -> RLModuleSpecType:
if self.framework_str == "torch":
from ray.rllib.algorithms.iql.torch.default_iql_torch_rl_module import (
DefaultIQLTorchRLModule,
)
return RLModuleSpec(module_class=DefaultIQLTorchRLModule)
else:
raise ValueError(
f"The framework {self.framework_str} is not supported. "
"Use `torch` instead."
)
@override(MARWILConfig)
def build_learner_connector(
self,
input_observation_space,
input_action_space,
device=None,
):
pipeline = super().build_learner_connector(
input_observation_space=input_observation_space,
input_action_space=input_action_space,
device=device,
)
# Remove unneeded connectors from the MARWIL connector pipeline.
pipeline.remove("AddOneTsToEpisodesAndTruncate")
pipeline.remove("GeneralAdvantageEstimation")
# Prepend the "add-NEXT_OBS-from-episodes-to-train-batch" connector piece (right
# after the corresponding "add-OBS-..." default piece).
pipeline.insert_after(
AddObservationsFromEpisodesToBatch,
AddNextObservationsFromEpisodesToTrainBatch(),
)
return pipeline
[docs]
@override(MARWILConfig)
def validate(self) -> None:
# Call super's validation method.
super().validate()
# Ensure hyperparameters are meaningful.
if self.beta <= 0.0:
self._value_error(
"For meaningful results, `beta` (temperature) parameter must be >> 0.0!"
)
if not 0.0 < self.expectile < 1.0:
self._value_error(
"For meaningful results, `expectile` parameter must be in (0, 1)."
)
@property
def _model_config_auto_includes(self):
return super()._model_config_auto_includes | {"twin_q": self.twin_q}
class IQL(MARWIL):
"""Implicit Q-learning (derived from MARWIL).
Uses MARWIL training step.
"""
@classmethod
@override(MARWIL)
def get_default_config(cls) -> AlgorithmConfig:
return IQLConfig()