RLLibのパラメータ、使い方など

参考：# https://dmv.nico/ja/articles/how_to_use_rllib/

QuickStart

基本的にはtune.Experimentに設定を書き、tune.run_experiments()を使うのが良い

import argparse
import gym
from gym.spaces import Discrete, Box
import numpy as np
import os
import random
import cv2

import ray
from ray.tune import Experiment
from ray import tune
from ray.rllib.agents import ppo
from ray.rllib.env.env_context import EnvContext
from ray.rllib.models import ModelCatalog
from ray.rllib.models.torch.torch_modelv2 import TorchModelV2
from ray.rllib.models.torch.fcnet import FullyConnectedNetwork as TorchFC
from ray.rllib.utils.framework import try_import_tf, try_import_torch
from ray.rllib.utils.test_utils import check_learning_achieved
from ray.tune.logger import pretty_print
from torch._C import dtype

if __name__ == "__main__":
    ray.init()

    # https://docs.ray.io/en/latest/rllib-training.html#specifying-resources
    experiment_spec = Experiment(
        "my_experiment_name",
        run="DQN",
        stop={
            "training_iteration": 100,
            #"timesteps_total": 100000,
            #"episode_reward_mean": 10,
        },
        config={
            "env": "PongDeterministic-v4",
            "env_config": {},
            "framework": "torch",
        },
        local_dir="~/ray_results",
        checkpoint_freq=10,
        max_failures=2
    )

    # automated run with Tune and grid search and TensorBoard
    print("Training automatically with Ray Tune")
    tune.run_experiments(experiment_spec)
    ray.shutdown()
    print("Finished")

EnvとModelをカスタマイズする場合

class SimpleCorridor(gym.Env):
    """Example of a custom env in which you have to walk down a corridor.
    You can configure the length of the corridor via the env config."""

    def __init__(self, config: EnvContext):
        self.end_pos = config["corridor_length"]
        self.cur_pos = 0
        self.action_space = Discrete(2)
        self.observation_space = Box(
            0.0, self.end_pos, shape=(1, ), dtype=np.float32)
        # Set the seed. This is only used for the final (reach goal) reward.
        self.seed(config.worker_index * config.num_workers)

    def reset(self):
        self.cur_pos = 0
        return np.array([self.cur_pos], dtype=np.float32)

    def step(self, action):
        assert action in [0, 1], action
        if action == 0 and self.cur_pos > 0:
            self.cur_pos -= 1
        elif action == 1:
            self.cur_pos += 1
        done = self.cur_pos >= self.end_pos
        # Produce a random reward when we reach the goal.
        return np.array([self.cur_pos], dtype=np.float32), \
            random.random() * 2 if done else -0.1, done, {}

    def seed(self, seed=None):
        random.seed(seed)


class TorchCustomModel(TorchModelV2, nn.Module):
    """Example of a PyTorch custom model that just delegates to a fc-net."""

    def __init__(self, obs_space, action_space, num_outputs, model_config,
                 name):
        TorchModelV2.__init__(self, obs_space, action_space, num_outputs,
                              model_config, name)
        nn.Module.__init__(self)
        print("obsspace: ", obs_space, action_space, model_config)
        self.torch_sub_model = TorchFC(obs_space, action_space, num_outputs,
                                       model_config, name)

    def forward(self, input_dict, state, seq_lens):
        input_dict["obs"] = input_dict["obs"].float()
        fc_out, _ = self.torch_sub_model(input_dict, state, seq_lens)
        return fc_out, []

    def value_function(self):
        return torch.reshape(self.torch_sub_model.value_function(), [-1])

if __name__ == "__main__":
    ray.init()
    ModelCatalog.register_custom_model(
        "my_model", TorchCustomModel
    )
    # https://docs.ray.io/en/latest/rllib-training.html#specifying-resources
    experiment_spec = Experiment(
        "my_experiment_name",
        run="DQN",
        stop={
            "training_iteration": 100,
            #"timesteps_total": 100000,
            #"episode_reward_mean": 10,
        },
        config={
            "env": SimpleCorridor,
            "env_config": {
                "corridor_length": 5,
            },
            "model": {
                "custom_model": "my_model",
            },
            "framework": "torch",
        },
        local_dir="~/ray_results",
        checkpoint_freq=10,
        max_failures=2
    )

    # automated run with Tune and grid search and TensorBoard
    print("Training automatically with Ray Tune")
    tune.run_experiments(experiment_spec)
    ray.shutdown()
    print("Finished")

DQNのパラメーターとRainbowの実装について

詳しいパラメータ内容はhttps://docs.ray.io/en/latest/rllib-algorithms.html#dqn

DEFAULT_CONFIG = Trainer.merge_trainer_configs(
    SIMPLEQ_DEFAULT_CONFIG,
    {
        # === Model ===
        # Number of atoms for representing the distribution of return. When
        # this is greater than 1, distributional Q-learning is used.
        # the discrete supports are bounded by v_min and v_max
        "num_atoms": 1,
        "v_min": -10.0,
        "v_max": 10.0,
        # Whether to use noisy network
        "noisy": False,
        # control the initial value of noisy nets
        "sigma0": 0.5,
        # Whether to use dueling dqn
        "dueling": True,
        # Dense-layer setup for each the advantage branch and the value branch
        # in a dueling architecture.
        "hiddens": [256],
        # Whether to use double dqn
        "double_q": True,
        # N-step Q learning
        "n_step": 1,

        # === Prioritized replay buffer ===
        # If True prioritized replay buffer will be used.
        "prioritized_replay": True,
        # Alpha parameter for prioritized replay buffer.
        "prioritized_replay_alpha": 0.6,
        # Beta parameter for sampling from prioritized replay buffer.
        "prioritized_replay_beta": 0.4,
        # Final value of beta (by default, we use constant beta=0.4).
        "final_prioritized_replay_beta": 0.4,
        # Time steps over which the beta parameter is annealed.
        "prioritized_replay_beta_annealing_timesteps": 20000,
        # Epsilon to add to the TD errors when updating priorities.
        "prioritized_replay_eps": 1e-6,

        # Callback to run before learning on a multi-agent batch of
        # experiences.
        "before_learn_on_batch": None,

        # The intensity with which to update the model (vs collecting samples
        # from the env). If None, uses the "natural" value of:
        # `train_batch_size` / (`rollout_fragment_length` x `num_workers` x
        # `num_envs_per_worker`).
        # If provided, will make sure that the ratio between ts inserted into
        # and sampled from the buffer matches the given value.
        # Example:
        #   training_intensity=1000.0
        #   train_batch_size=250 rollout_fragment_length=1
        #   num_workers=1 (or 0) num_envs_per_worker=1
        #   -> natural value = 250 / 1 = 250.0
        #   -> will make sure that replay+train op will be executed 4x as
        #      often as rollout+insert op (4 * 250 = 1000).
        # See: rllib/agents/dqn/dqn.py::calculate_rr_weights for further
        # details.
        "training_intensity": None,

        # === Parallelism ===
        # Whether to compute priorities on workers.
        "worker_side_prioritization": False,
    },
    _allow_unknown_configs=True,
)

DuelingNetworkとDoubleQNetworkはデフォルトでONになっていることに注意

Rainbow

Rainbowにするには以下のように変更する必要がある

For a complete rainbow setup, make the following changes to the default DQN config: "n_step": [between 1 and 10], "noisy": True, "num_atoms": [more than 1], "v_min": -10.0, "v_max": 10.0 (set v_min and v_max according to your expected range of returns).

config全体としては以下のようになる

atari-basic-dqn:
    env:
        grid_search:
            - BreakoutNoFrameskip-v4
            - BeamRiderNoFrameskip-v4
            - QbertNoFrameskip-v4
            - SpaceInvadersNoFrameskip-v4
    run: DQN
    config:
        framework: torch
        double_q: true
        dueling: true
        num_atoms: 51
        noisy: true
        gamma: 0.99
        lr: .0001
        hiddens: [512]
        learning_starts: 10000
        buffer_size: 50000
        rollout_fragment_length: 4
        train_batch_size: 32
        exploration_config:
          epsilon_timesteps: 2
          final_epsilon: 0.0
        target_network_update_freq: 500
        prioritized_replay: True
        prioritized_replay_alpha: 0.5
        final_prioritized_replay_beta: 1.0
        prioritized_replay_beta_annealing_timesteps: 400000
        n_step: 3
        gpu: True