Source code for ray.train.tensorflow.tensorflow_trainer

from typing import Any, Callable, Dict, Optional, Union

from ray.train import Checkpoint, DataConfig, RunConfig, ScalingConfig
from ray.train.data_parallel_trainer import DataParallelTrainer
from ray.train.tensorflow.config import TensorflowConfig
from ray.train.trainer import GenDataset
from ray.util import PublicAPI


[docs] @PublicAPI(stability="beta") class TensorflowTrainer(DataParallelTrainer): """A Trainer for data parallel Tensorflow training. This Trainer runs the function ``train_loop_per_worker`` on multiple Ray Actors. These actors already have the necessary TensorFlow process group already configured for distributed TensorFlow training. If ``train_loop_per_worker`` accepts an argument, then ``train_loop_config`` will be passed in as the argument. This is useful if you want to tune the values in ``train_loop_config`` as hyperparameters. If the ``datasets`` dict contains a training dataset (denoted by the "train" key), then it will be split into multiple dataset shards that can then be accessed by ``ray.train.get_dataset_shard("train")`` inside ``train_loop_per_worker``. All the other datasets will not be split and ``ray.train.get_dataset_shard(...)`` will return the entire Dataset. .. warning:: Ray will not automatically set any environment variables or configuration related to local parallelism / threading :ref:`aside from "OMP_NUM_THREADS" <omp-num-thread-note>`. If you desire greater control over TensorFlow threading, use the ``tf.config.threading`` module (eg. ``tf.config.threading.set_inter_op_parallelism_threads(num_cpus)``) at the beginning of your ``train_loop_per_worker`` function. Any returns from the ``train_loop_per_worker`` will be discarded and not used or persisted anywhere. Examples: The ``train_loop_per_worker`` function is expected to take in either 0 or 1 arguments: >>> def train_loop_per_worker(): ... ... >>> def train_loop_per_worker(config: Dict): ... ... Inside the ``train_loop_per_worker`` function, you can use any of the :ref:`Ray Train loop methods <train-loop-api>`. >>> from ray import train >>> def train_loop_per_worker(): ... # Report intermediate results for callbacks or logging and ... # checkpoint data. ... train.report(...) ... ... # Returns the last saved checkpoint, or None. ... train.get_checkpoint() ... ... # Returns the Dataset shard for the given key. ... train.get_dataset_shard("my_dataset") ... ... # Returns the total number of workers executing training. ... train.get_context().get_world_size() ... ... # Returns the rank of this worker. ... train.get_context().get_world_rank() ... ... # Returns the rank of the worker on the current node. ... train.get_context().get_local_rank() >>> import os # doctest: +SKIP >>> import tempfile # doctest: +SKIP >>> import tensorflow as tf # doctest: +SKIP >>> import numpy as np # doctest: +SKIP >>> import ray # doctest: +SKIP >>> from ray import train # doctest: +SKIP >>> from ray.train import Checkpoint, ScalingConfig # doctest: +SKIP >>> from ray.train.tensorflow import TensorflowTrainer # doctest: +SKIP >>> def build_model(): # doctest: +SKIP ... # toy neural network : 1-layer ... return tf.keras.Sequential( ... [tf.keras.layers.Dense( ... 1, activation="linear", input_shape=(1,))] ... ) >>> def train_loop_per_worker(config): # doctest: +SKIP ... dataset_shard = train.get_dataset_shard("train") ... strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy() ... with strategy.scope(): ... model = build_model() ... model.compile( ... optimizer="Adam", loss="mean_squared_error", metrics=["mse"]) ... ... tf_dataset = dataset_shard.to_tf( ... feature_columns="x", ... label_columns="y", ... batch_size=1 ... ) ... for epoch in range(config["num_epochs"]): ... model.fit(tf_dataset) ... ... # Create checkpoint. ... checkpoint_dir = tempfile.mkdtemp() ... model.save_weights( ... os.path.join(checkpoint_dir, "my_checkpoint") ... ) ... checkpoint = Checkpoint.from_directory(checkpoint_dir) ... ... train.report( ... {}, ... checkpoint=checkpoint, ... ) >>> train_dataset = ray.data.from_items([{"x": np.array([x], dtype=np.float32), "y": x + 1} for x in range(32)]) # doctest: +SKIP >>> trainer = TensorflowTrainer( # doctest: +SKIP ... train_loop_per_worker=train_loop_per_worker, ... scaling_config=ScalingConfig(num_workers=3, use_gpu=True), ... datasets={"train": train_dataset}, ... train_loop_config={"num_epochs": 2}, ... ) >>> result = trainer.fit() # doctest: +SKIP Args: train_loop_per_worker: The training function to execute. This can either take in no arguments or a ``config`` dict. train_loop_config: Configurations to pass into ``train_loop_per_worker`` if it accepts an argument. tensorflow_config: Configuration for setting up the TensorFlow backend. If set to None, use the default configuration. This replaces the ``backend_config`` arg of ``DataParallelTrainer``. scaling_config: Configuration for how to scale data parallel training. dataset_config: Configuration for dataset ingest. run_config: Configuration for the execution of the training run. datasets: Any Datasets to use for training. Use the key "train" to denote which dataset is the training dataset. metadata: Dict that should be made available via `ray.train.get_context().get_metadata()` and in `checkpoint.get_metadata()` for checkpoints saved from this Trainer. Must be JSON-serializable. resume_from_checkpoint: A checkpoint to resume training from. """ def __init__( self, train_loop_per_worker: Union[Callable[[], None], Callable[[Dict], None]], *, train_loop_config: Optional[Dict] = None, tensorflow_config: Optional[TensorflowConfig] = None, scaling_config: Optional[ScalingConfig] = None, dataset_config: Optional[DataConfig] = None, run_config: Optional[RunConfig] = None, datasets: Optional[Dict[str, GenDataset]] = None, metadata: Optional[Dict[str, Any]] = None, resume_from_checkpoint: Optional[Checkpoint] = None, ): if not tensorflow_config: tensorflow_config = TensorflowConfig() super(TensorflowTrainer, self).__init__( train_loop_per_worker=train_loop_per_worker, train_loop_config=train_loop_config, backend_config=tensorflow_config, scaling_config=scaling_config, dataset_config=dataset_config, run_config=run_config, datasets=datasets, resume_from_checkpoint=resume_from_checkpoint, metadata=metadata, )