Source code for ray.rllib.algorithms.bc.bc
from ray.rllib.algorithms.algorithm_config import AlgorithmConfig
from ray.rllib.algorithms.marwil.marwil import MARWIL, MARWILConfig
from ray.rllib.core.rl_module.rl_module import RLModuleSpec
from ray.rllib.utils.annotations import override
from ray.rllib.utils.typing import RLModuleSpecType
[docs]
class BCConfig(MARWILConfig):
"""Defines a configuration class from which a new BC Algorithm can be built
Examples:
>>> from pathlib import Path
>>> import gymnasium as gym
>>> from ray.rllib.algorithms.bc import BCConfig
>>> # Get the path to the CartPole data in the rllib folder.
>>> base_path = Path(__file__).parents[2]
>>> data_path = base_path / "offline/tests/data/cartpole/cartpole-v1_large"
>>> config = BCConfig().training(lr=0.00001, gamma=0.99)
>>> # Set the observation and action spaces of the data.
>>> env = gym.make("CartPole-v1")
>>> config = config.environment(
... observation_space=env.observation_space,
... action_space=env.action_space,
... )
>>> config = config.offline_data(
... input_=[data_path.as_posix()],
... # Run exactly one update per training iteration.
... dataset_num_iters_per_learner=1,
... )
>>> # Build an Algorithm object from the config and run 1 training iteration.
>>> algo = config.build()
>>> _ = algo.train()
>>> from ray.rllib.algorithms.bc import BCConfig
>>> from ray import tune
>>> config = BCConfig()
>>> # Print out some default values.
>>> print(config.beta)
0.0
>>> # Update the config object.
>>> _ = config.training(lr=tune.grid_search([0.001, 0.0001]))
>>> # Set the config object's data path.
>>> _ = config.offline_data(
... input_=[data_path.as_posix()],
... dataset_num_iters_per_learner=1,
... )
>>> # Set the observation and action spaces of the data.
>>> _ = config.environment(
... observation_space=env.observation_space,
... action_space=env.action_space,
... )
>>> # Use to_dict() to get the old-style python config dict
>>> # when running with tune.
>>> print("[ignore]"); results = tune.Tuner(
... "BC",
... param_space=config.to_dict(),
... # Run one trial at a time so each trial's offline data
... # pipeline can get the CPUs it needs.
... tune_config=tune.TuneConfig(max_concurrent_trials=1),
... run_config=tune.RunConfig(stop={"training_iteration": 1}),
... ).fit() # doctest: +ELLIPSIS
[ignore]...
>>> results.num_errors
0
"""
def __init__(self, algo_class=None):
"""Initializes a BCConfig instance."""
super().__init__(algo_class=algo_class or BC)
# fmt: off
# __sphinx_doc_begin__
# No need to calculate advantages (or do anything else with the rewards).
self.beta = 0.0
# Advantages (calculated during postprocessing)
# not important for behavioral cloning.
self.postprocess_inputs = False
# Materialize only the mapped data. This is optimal as long
# as no connector in the connector pipeline holds a state.
self.materialize_data = False
self.materialize_mapped_data = True
# __sphinx_doc_end__
# fmt: on
[docs]
@override(AlgorithmConfig)
def get_default_rl_module_spec(self) -> RLModuleSpecType:
if self.framework_str == "torch":
from ray.rllib.algorithms.bc.torch.default_bc_torch_rl_module import (
DefaultBCTorchRLModule,
)
return RLModuleSpec(module_class=DefaultBCTorchRLModule)
else:
raise ValueError(
f"The framework {self.framework_str} is not supported. "
"Use `torch` instead."
)
@override(AlgorithmConfig)
def build_learner_connector(
self,
input_observation_space,
input_action_space,
device=None,
):
pipeline = super().build_learner_connector(
input_observation_space=input_observation_space,
input_action_space=input_action_space,
device=device,
)
# Remove unneeded connectors from the MARWIL connector pipeline.
pipeline.remove("AddOneTsToEpisodesAndTruncate")
pipeline.remove("GeneralAdvantageEstimation")
return pipeline
[docs]
@override(MARWILConfig)
def validate(self) -> None:
# Call super's validation method.
super().validate()
if self.beta != 0.0:
self._value_error("For behavioral cloning, `beta` parameter must be 0.0!")
class BC(MARWIL):
"""Behavioral Cloning (derived from MARWIL).
Uses MARWIL with beta force-set to 0.0.
"""
@classmethod
@override(MARWIL)
def get_default_config(cls) -> BCConfig:
return BCConfig()