diff --git a/unit8_part2.ipynb b/unit8_part2.ipynb index f634a9e..d89a8e1 100644 --- a/unit8_part2.ipynb +++ b/unit8_part2.ipynb @@ -444,27 +444,7 @@ }, { "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Namespace(help=False, algo='APPO', env='doom_health_gathering_supreme', experiment='default_experiment', train_dir='/Users/Yang/Desktop/course-deepRL-huggingface/unit8-part2-doom/train_dir', restart_behavior='resume', device='gpu', seed=None, num_policies=1, async_rl=True, serial_mode=False, batched_sampling=False, num_batches_to_accumulate=2, worker_num_splits=2, policy_workers_per_policy=1, max_policy_lag=1000, num_workers=8, num_envs_per_worker=4, batch_size=1024, num_batches_per_epoch=1, num_epochs=1, rollout=32, recurrence=-1, shuffle_minibatches=False, gamma=0.99, reward_scale=1.0, reward_clip=1000.0, value_bootstrap=False, normalize_returns=True, exploration_loss_coeff=0.001, value_loss_coeff=0.5, kl_loss_coeff=0.0, exploration_loss='symmetric_kl', gae_lambda=0.95, ppo_clip_ratio=0.1, ppo_clip_value=0.2, with_vtrace=False, vtrace_rho=1.0, vtrace_c=1.0, optimizer='adam', adam_eps=1e-06, adam_beta1=0.9, adam_beta2=0.999, max_grad_norm=4.0, learning_rate=0.0001, lr_schedule='constant', lr_schedule_kl_threshold=0.008, lr_adaptive_min=1e-06, lr_adaptive_max=0.01, obs_subtract_mean=0.0, obs_scale=255.0, normalize_input=True, normalize_input_keys=None, decorrelate_experience_max_seconds=0, decorrelate_envs_on_one_worker=True, actor_worker_gpus=[], set_workers_cpu_affinity=True, force_envs_single_thread=False, default_niceness=0, log_to_file=True, experiment_summaries_interval=10, flush_summaries_interval=30, stats_avg=100, summaries_use_frameskip=True, heartbeat_interval=20, heartbeat_reporting_interval=600, train_for_env_steps=4000000, train_for_seconds=10000000000, save_every_sec=120, keep_checkpoints=2, load_checkpoint_kind='latest', save_milestones_sec=-1, save_best_every_sec=5, save_best_metric='reward', save_best_after=100000, benchmark=False, encoder_mlp_layers=[512, 512], encoder_conv_architecture='convnet_simple', encoder_conv_mlp_layers=[512], use_rnn=True, rnn_size=512, rnn_type='gru', rnn_num_layers=1, decoder_mlp_layers=[], nonlinearity='elu', policy_initialization='orthogonal', policy_init_gain=1.0, actor_critic_share_weights=True, adaptive_stddev=True, continuous_tanh_scale=0.0, initial_stddev=1.0, use_env_info_cache=False, env_gpu_actions=False, env_gpu_observations=True, env_frameskip=4, env_framestack=1, pixel_format='CHW', use_record_episode_statistics=False, with_wandb=False, wandb_user=None, wandb_project='sample_factory', wandb_group=None, wandb_job_type='SF', wandb_tags=[], with_pbt=False, pbt_mix_policies_in_one_env=True, pbt_period_env_steps=5000000, pbt_start_mutation=20000000, pbt_replace_fraction=0.3, pbt_mutation_rate=0.15, pbt_replace_reward_gap=0.1, pbt_replace_reward_gap_absolute=1e-06, pbt_optimize_gamma=False, pbt_target_objective='true_objective', pbt_perturb_min=1.1, pbt_perturb_max=1.5, num_agents=-1, num_humans=0, num_bots=-1, start_bot_difficulty=None, timelimit=None, res_w=128, res_h=72, wide_aspect_ratio=False, eval_env_frameskip=1, fps=35, command_line='--env=doom_health_gathering_supreme --num_workers=8 --num_envs_per_worker=4 --train_for_env_steps=4000000', cli_args={'env': 'doom_health_gathering_supreme', 'num_workers': 8, 'num_envs_per_worker': 4, 'train_for_env_steps': 4000000}, git_hash='unknown', git_repo_name='not a git repository')" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "cfg" - ] - }, - { - "cell_type": "code", - "execution_count": 13, + "execution_count": null, "metadata": { "id": "y_TeicMvyKHP" }, @@ -473,182 +453,143 @@ "name": "stderr", "output_type": "stream", "text": [ - "\u001b[36m[2025-06-01 20:17:57,422][62189] register_encoder_factory: \u001b[0m\n", - "\u001b[33m[2025-06-01 20:17:57,432][62189] Saved parameter configuration for experiment default_experiment not found!\u001b[0m\n", - "\u001b[33m[2025-06-01 20:17:57,432][62189] Starting experiment from scratch!\u001b[0m\n", - "\u001b[36m[2025-06-01 20:17:57,602][62189] Experiment dir /Users/Yang/Desktop/course-deepRL-huggingface/unit8-part2-doom/train_dir/default_experiment already exists!\u001b[0m\n", - "\u001b[36m[2025-06-01 20:17:57,603][62189] Resuming existing experiment from /Users/Yang/Desktop/course-deepRL-huggingface/unit8-part2-doom/train_dir/default_experiment...\u001b[0m\n", - "\u001b[36m[2025-06-01 20:17:57,603][62189] Weights and Biases integration disabled\u001b[0m\n", - "\u001b[36m[2025-06-01 20:17:58,601][62189] Queried available GPUs: \n", - "\u001b[0m\n", - "\u001b[37m\u001b[1m[2025-06-01 20:17:58,603][62189] Environment var CUDA_VISIBLE_DEVICES is \n", - "\u001b[0m\n", - "\n", - "\u001b[36m[2025-06-01 20:17:59,529][62902] Doom resolution: 160x120, resize resolution: (128, 72)\u001b[0m\n", - "/Users/Yang/Desktop/course-deepRL-huggingface/unit8-part2-doom/.conda/lib/python3.10/site-packages/gymnasium/core.py:311: UserWarning: \u001b[33mWARN: env.num_agents to get variables from other wrappers is deprecated and will be removed in v1.0, to get this variable you can do `env.unwrapped.num_agents` for environment variables or `env.get_wrapper_attr('num_agents')` that will search the reminding wrappers.\u001b[0m\n", - " logger.warn(\n", - "/Users/Yang/Desktop/course-deepRL-huggingface/unit8-part2-doom/.conda/lib/python3.10/site-packages/gymnasium/core.py:311: UserWarning: \u001b[33mWARN: env.is_multiagent to get variables from other wrappers is deprecated and will be removed in v1.0, to get this variable you can do `env.unwrapped.is_multiagent` for environment variables or `env.get_wrapper_attr('is_multiagent')` that will search the reminding wrappers.\u001b[0m\n", - " logger.warn(\n", - "\u001b[36m[2025-06-01 20:17:59,530][62902] Env info: EnvInfo(obs_space=Dict('obs': Box(0, 255, (3, 72, 128), uint8)), action_space=Discrete(5), num_agents=1, gpu_actions=False, gpu_observations=True, action_splits=None, all_discrete=None, frameskip=4, reward_shaping_scheme=None, env_info_protocol_version=1)\u001b[0m\n", - "\u001b[36m[2025-06-01 20:17:59,703][62189] Automatically setting recurrence to 32\u001b[0m\n", - "\u001b[36m[2025-06-01 20:17:59,703][62189] Starting experiment with the following configuration:\n", - "help=False\n", - "algo=APPO\n", - "env=doom_health_gathering_supreme\n", - "experiment=default_experiment\n", - "train_dir=/Users/Yang/Desktop/course-deepRL-huggingface/unit8-part2-doom/train_dir\n", - "restart_behavior=resume\n", - "device=gpu\n", - "seed=None\n", - "num_policies=1\n", - "async_rl=True\n", - "serial_mode=False\n", - "batched_sampling=False\n", - "num_batches_to_accumulate=2\n", - "worker_num_splits=2\n", - "policy_workers_per_policy=1\n", - "max_policy_lag=1000\n", - "num_workers=8\n", - "num_envs_per_worker=4\n", - "batch_size=1024\n", - "num_batches_per_epoch=1\n", - "num_epochs=1\n", - "rollout=32\n", - "recurrence=32\n", - "shuffle_minibatches=False\n", - "gamma=0.99\n", - "reward_scale=1.0\n", - "reward_clip=1000.0\n", - "value_bootstrap=False\n", - "normalize_returns=True\n", - "exploration_loss_coeff=0.001\n", - "value_loss_coeff=0.5\n", - "kl_loss_coeff=0.0\n", - "exploration_loss=symmetric_kl\n", - "gae_lambda=0.95\n", - "ppo_clip_ratio=0.1\n", - "ppo_clip_value=0.2\n", - "with_vtrace=False\n", - "vtrace_rho=1.0\n", - "vtrace_c=1.0\n", - "optimizer=adam\n", - "adam_eps=1e-06\n", - "adam_beta1=0.9\n", - "adam_beta2=0.999\n", - "max_grad_norm=4.0\n", - "learning_rate=0.0001\n", - "lr_schedule=constant\n", - "lr_schedule_kl_threshold=0.008\n", - "lr_adaptive_min=1e-06\n", - "lr_adaptive_max=0.01\n", - "obs_subtract_mean=0.0\n", - "obs_scale=255.0\n", - "normalize_input=True\n", - "normalize_input_keys=None\n", - "decorrelate_experience_max_seconds=0\n", - "decorrelate_envs_on_one_worker=True\n", - "actor_worker_gpus=[]\n", - "set_workers_cpu_affinity=True\n", - "force_envs_single_thread=False\n", - "default_niceness=0\n", - "log_to_file=True\n", - "experiment_summaries_interval=10\n", - "flush_summaries_interval=30\n", - "stats_avg=100\n", - "summaries_use_frameskip=True\n", - "heartbeat_interval=20\n", - "heartbeat_reporting_interval=600\n", - "train_for_env_steps=4000000\n", - "train_for_seconds=10000000000\n", - "save_every_sec=120\n", - "keep_checkpoints=2\n", - "load_checkpoint_kind=latest\n", - "save_milestones_sec=-1\n", - "save_best_every_sec=5\n", - "save_best_metric=reward\n", - "save_best_after=100000\n", - "benchmark=False\n", - "encoder_mlp_layers=[512, 512]\n", - "encoder_conv_architecture=convnet_simple\n", - "encoder_conv_mlp_layers=[512]\n", - "use_rnn=True\n", - "rnn_size=512\n", - "rnn_type=gru\n", - "rnn_num_layers=1\n", - "decoder_mlp_layers=[]\n", - "nonlinearity=elu\n", - "policy_initialization=orthogonal\n", - "policy_init_gain=1.0\n", - "actor_critic_share_weights=True\n", - "adaptive_stddev=True\n", - "continuous_tanh_scale=0.0\n", - "initial_stddev=1.0\n", - "use_env_info_cache=False\n", - "env_gpu_actions=False\n", - "env_gpu_observations=True\n", - "env_frameskip=4\n", - "env_framestack=1\n", - "pixel_format=CHW\n", - "use_record_episode_statistics=False\n", - "with_wandb=False\n", - "wandb_user=None\n", - "wandb_project=sample_factory\n", - "wandb_group=None\n", - "wandb_job_type=SF\n", - "wandb_tags=[]\n", - "with_pbt=False\n", - "pbt_mix_policies_in_one_env=True\n", - "pbt_period_env_steps=5000000\n", - "pbt_start_mutation=20000000\n", - "pbt_replace_fraction=0.3\n", - "pbt_mutation_rate=0.15\n", - "pbt_replace_reward_gap=0.1\n", - "pbt_replace_reward_gap_absolute=1e-06\n", - "pbt_optimize_gamma=False\n", - "pbt_target_objective=true_objective\n", - "pbt_perturb_min=1.1\n", - "pbt_perturb_max=1.5\n", - "num_agents=-1\n", - "num_humans=0\n", - "num_bots=-1\n", - "start_bot_difficulty=None\n", - "timelimit=None\n", - "res_w=128\n", - "res_h=72\n", - "wide_aspect_ratio=False\n", - "eval_env_frameskip=1\n", - "fps=35\n", - "command_line=--env=doom_health_gathering_supreme --num_workers=8 --num_envs_per_worker=4 --train_for_env_steps=4000000\n", - "cli_args={'env': 'doom_health_gathering_supreme', 'num_workers': 8, 'num_envs_per_worker': 4, 'train_for_env_steps': 4000000}\n", - "git_hash=unknown\n", - "git_repo_name=not a git repository\u001b[0m\n", - "\u001b[36m[2025-06-01 20:17:59,704][62189] Saving configuration to /Users/Yang/Desktop/course-deepRL-huggingface/unit8-part2-doom/train_dir/default_experiment/config.json...\u001b[0m\n", - "\u001b[36m[2025-06-01 20:17:59,705][62189] Rollout worker 0 uses device cpu\u001b[0m\n", - "\u001b[36m[2025-06-01 20:17:59,706][62189] Rollout worker 1 uses device cpu\u001b[0m\n", - "\u001b[36m[2025-06-01 20:17:59,706][62189] Rollout worker 2 uses device cpu\u001b[0m\n", - "\u001b[36m[2025-06-01 20:17:59,706][62189] Rollout worker 3 uses device cpu\u001b[0m\n", - "\u001b[36m[2025-06-01 20:17:59,707][62189] Rollout worker 4 uses device cpu\u001b[0m\n", - "\u001b[36m[2025-06-01 20:17:59,707][62189] Rollout worker 5 uses device cpu\u001b[0m\n", - "\u001b[36m[2025-06-01 20:17:59,707][62189] Rollout worker 6 uses device cpu\u001b[0m\n", - "\u001b[36m[2025-06-01 20:17:59,708][62189] Rollout worker 7 uses device cpu\u001b[0m\n" + "\u001b[33m[2025-06-01 20:20:30,096][62189] Environment doom_basic already registered, overwriting...\u001b[0m\n", + "\u001b[33m[2025-06-01 20:20:30,098][62189] Environment doom_two_colors_easy already registered, overwriting...\u001b[0m\n", + "\u001b[33m[2025-06-01 20:20:30,098][62189] Environment doom_two_colors_hard already registered, overwriting...\u001b[0m\n", + "\u001b[33m[2025-06-01 20:20:30,099][62189] Environment doom_dm already registered, overwriting...\u001b[0m\n", + "\u001b[33m[2025-06-01 20:20:30,099][62189] Environment doom_dwango5 already registered, overwriting...\u001b[0m\n", + "\u001b[33m[2025-06-01 20:20:30,099][62189] Environment doom_my_way_home_flat_actions already registered, overwriting...\u001b[0m\n", + "\u001b[33m[2025-06-01 20:20:30,100][62189] Environment doom_defend_the_center_flat_actions already registered, overwriting...\u001b[0m\n", + "\u001b[33m[2025-06-01 20:20:30,100][62189] Environment doom_my_way_home already registered, overwriting...\u001b[0m\n", + "\u001b[33m[2025-06-01 20:20:30,100][62189] Environment doom_deadly_corridor already registered, overwriting...\u001b[0m\n", + "\u001b[33m[2025-06-01 20:20:30,101][62189] Environment doom_defend_the_center already registered, overwriting...\u001b[0m\n", + "\u001b[33m[2025-06-01 20:20:30,101][62189] Environment doom_defend_the_line already registered, overwriting...\u001b[0m\n", + "\u001b[33m[2025-06-01 20:20:30,102][62189] Environment doom_health_gathering already registered, overwriting...\u001b[0m\n", + "\u001b[33m[2025-06-01 20:20:30,102][62189] Environment doom_health_gathering_supreme already registered, overwriting...\u001b[0m\n", + "\u001b[33m[2025-06-01 20:20:30,102][62189] Environment doom_battle already registered, overwriting...\u001b[0m\n", + "\u001b[33m[2025-06-01 20:20:30,103][62189] Environment doom_battle2 already registered, overwriting...\u001b[0m\n", + "\u001b[33m[2025-06-01 20:20:30,104][62189] Environment doom_duel_bots already registered, overwriting...\u001b[0m\n", + "\u001b[33m[2025-06-01 20:20:30,104][62189] Environment doom_deathmatch_bots already registered, overwriting...\u001b[0m\n", + "\u001b[33m[2025-06-01 20:20:30,105][62189] Environment doom_duel already registered, overwriting...\u001b[0m\n", + "\u001b[33m[2025-06-01 20:20:30,105][62189] Environment doom_deathmatch_full already registered, overwriting...\u001b[0m\n", + "\u001b[33m[2025-06-01 20:20:30,106][62189] Environment doom_benchmark already registered, overwriting...\u001b[0m\n", + "\u001b[36m[2025-06-01 20:20:30,106][62189] register_encoder_factory: \u001b[0m\n", + "usage: ipykernel_launcher.py [-h] [--algo ALGO] --env ENV\n", + " [--experiment EXPERIMENT] [--train_dir TRAIN_DIR]\n", + " [--restart_behavior {resume,restart,overwrite}]\n", + " [--device {gpu,cpu}] [--seed SEED]\n", + " [--num_policies NUM_POLICIES]\n", + " [--async_rl ASYNC_RL] [--serial_mode SERIAL_MODE]\n", + " [--batched_sampling BATCHED_SAMPLING]\n", + " [--num_batches_to_accumulate NUM_BATCHES_TO_ACCUMULATE]\n", + " [--worker_num_splits WORKER_NUM_SPLITS]\n", + " [--policy_workers_per_policy POLICY_WORKERS_PER_POLICY]\n", + " [--max_policy_lag MAX_POLICY_LAG]\n", + " [--num_workers NUM_WORKERS]\n", + " [--num_envs_per_worker NUM_ENVS_PER_WORKER]\n", + " [--batch_size BATCH_SIZE]\n", + " [--num_batches_per_epoch NUM_BATCHES_PER_EPOCH]\n", + " [--num_epochs NUM_EPOCHS] [--rollout ROLLOUT]\n", + " [--recurrence RECURRENCE]\n", + " [--shuffle_minibatches SHUFFLE_MINIBATCHES]\n", + " [--gamma GAMMA] [--reward_scale REWARD_SCALE]\n", + " [--reward_clip REWARD_CLIP]\n", + " [--value_bootstrap VALUE_BOOTSTRAP]\n", + " [--normalize_returns NORMALIZE_RETURNS]\n", + " [--exploration_loss_coeff EXPLORATION_LOSS_COEFF]\n", + " [--value_loss_coeff VALUE_LOSS_COEFF]\n", + " [--kl_loss_coeff KL_LOSS_COEFF]\n", + " [--exploration_loss {entropy,symmetric_kl}]\n", + " [--gae_lambda GAE_LAMBDA]\n", + " [--ppo_clip_ratio PPO_CLIP_RATIO]\n", + " [--ppo_clip_value PPO_CLIP_VALUE]\n", + " [--with_vtrace WITH_VTRACE]\n", + " [--vtrace_rho VTRACE_RHO] [--vtrace_c VTRACE_C]\n", + " [--optimizer {adam,lamb}] [--adam_eps ADAM_EPS]\n", + " [--adam_beta1 ADAM_BETA1]\n", + " [--adam_beta2 ADAM_BETA2]\n", + " [--max_grad_norm MAX_GRAD_NORM]\n", + " [--learning_rate LEARNING_RATE]\n", + " [--lr_schedule {constant,kl_adaptive_minibatch,kl_adaptive_epoch}]\n", + " [--lr_schedule_kl_threshold LR_SCHEDULE_KL_THRESHOLD]\n", + " [--lr_adaptive_min LR_ADAPTIVE_MIN]\n", + " [--lr_adaptive_max LR_ADAPTIVE_MAX]\n", + " [--obs_subtract_mean OBS_SUBTRACT_MEAN]\n", + " [--obs_scale OBS_SCALE]\n", + " [--normalize_input NORMALIZE_INPUT]\n", + " [--normalize_input_keys [NORMALIZE_INPUT_KEYS ...]]\n", + " [--decorrelate_experience_max_seconds DECORRELATE_EXPERIENCE_MAX_SECONDS]\n", + " [--decorrelate_envs_on_one_worker DECORRELATE_ENVS_ON_ONE_WORKER]\n", + " [--actor_worker_gpus [ACTOR_WORKER_GPUS ...]]\n", + " [--set_workers_cpu_affinity SET_WORKERS_CPU_AFFINITY]\n", + " [--force_envs_single_thread FORCE_ENVS_SINGLE_THREAD]\n", + " [--default_niceness DEFAULT_NICENESS]\n", + " [--log_to_file LOG_TO_FILE]\n", + " [--experiment_summaries_interval EXPERIMENT_SUMMARIES_INTERVAL]\n", + " [--flush_summaries_interval FLUSH_SUMMARIES_INTERVAL]\n", + " [--stats_avg STATS_AVG]\n", + " [--summaries_use_frameskip SUMMARIES_USE_FRAMESKIP]\n", + " [--heartbeat_interval HEARTBEAT_INTERVAL]\n", + " [--heartbeat_reporting_interval HEARTBEAT_REPORTING_INTERVAL]\n", + " [--train_for_env_steps TRAIN_FOR_ENV_STEPS]\n", + " [--train_for_seconds TRAIN_FOR_SECONDS]\n", + " [--save_every_sec SAVE_EVERY_SEC]\n", + " [--keep_checkpoints KEEP_CHECKPOINTS]\n", + " [--load_checkpoint_kind {latest,best}]\n", + " [--save_milestones_sec SAVE_MILESTONES_SEC]\n", + " [--save_best_every_sec SAVE_BEST_EVERY_SEC]\n", + " [--save_best_metric SAVE_BEST_METRIC]\n", + " [--save_best_after SAVE_BEST_AFTER]\n", + " [--benchmark BENCHMARK]\n", + " [--encoder_mlp_layers [ENCODER_MLP_LAYERS ...]]\n", + " [--encoder_conv_architecture {convnet_simple,convnet_impala,convnet_atari,resnet_impala}]\n", + " [--encoder_conv_mlp_layers [ENCODER_CONV_MLP_LAYERS ...]]\n", + " [--use_rnn USE_RNN] [--rnn_size RNN_SIZE]\n", + " [--rnn_type {gru,lstm}]\n", + " [--rnn_num_layers RNN_NUM_LAYERS]\n", + " [--decoder_mlp_layers [DECODER_MLP_LAYERS ...]]\n", + " [--nonlinearity {elu,relu,tanh}]\n", + " [--policy_initialization {orthogonal,xavier_uniform,torch_default}]\n", + " [--policy_init_gain POLICY_INIT_GAIN]\n", + " [--actor_critic_share_weights ACTOR_CRITIC_SHARE_WEIGHTS]\n", + " [--adaptive_stddev ADAPTIVE_STDDEV]\n", + " [--continuous_tanh_scale CONTINUOUS_TANH_SCALE]\n", + " [--initial_stddev INITIAL_STDDEV]\n", + " [--use_env_info_cache USE_ENV_INFO_CACHE]\n", + " [--env_gpu_actions ENV_GPU_ACTIONS]\n", + " [--env_gpu_observations ENV_GPU_OBSERVATIONS]\n", + " [--env_frameskip ENV_FRAMESKIP]\n", + " [--env_framestack ENV_FRAMESTACK]\n", + " [--pixel_format PIXEL_FORMAT]\n", + " [--use_record_episode_statistics USE_RECORD_EPISODE_STATISTICS]\n", + " [--with_wandb WITH_WANDB]\n", + " [--wandb_user WANDB_USER]\n", + " [--wandb_project WANDB_PROJECT]\n", + " [--wandb_group WANDB_GROUP]\n", + " [--wandb_job_type WANDB_JOB_TYPE]\n", + " [--wandb_tags [WANDB_TAGS ...]]\n", + " [--with_pbt WITH_PBT]\n", + " [--pbt_mix_policies_in_one_env PBT_MIX_POLICIES_IN_ONE_ENV]\n", + " [--pbt_period_env_steps PBT_PERIOD_ENV_STEPS]\n", + " [--pbt_start_mutation PBT_START_MUTATION]\n", + " [--pbt_replace_fraction PBT_REPLACE_FRACTION]\n", + " [--pbt_mutation_rate PBT_MUTATION_RATE]\n", + " [--pbt_replace_reward_gap PBT_REPLACE_REWARD_GAP]\n", + " [--pbt_replace_reward_gap_absolute PBT_REPLACE_REWARD_GAP_ABSOLUTE]\n", + " [--pbt_optimize_gamma PBT_OPTIMIZE_GAMMA]\n", + " [--pbt_target_objective PBT_TARGET_OBJECTIVE]\n", + " [--pbt_perturb_min PBT_PERTURB_MIN]\n", + " [--pbt_perturb_max PBT_PERTURB_MAX]\n", + "ipykernel_launcher.py: error: argument --device: invalid choice: \"'gpu'--num_envs_per_worker=2\" (choose from 'gpu', 'cpu')\n" ] }, { - "ename": "IndexError", - "evalue": "list index out of range", + "ename": "SystemExit", + "evalue": "2", "output_type": "error", "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mIndexError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[13], line 9\u001b[0m\n\u001b[1;32m 6\u001b[0m env \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdoom_health_gathering_supreme\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 7\u001b[0m cfg \u001b[38;5;241m=\u001b[39m parse_vizdoom_cfg(argv\u001b[38;5;241m=\u001b[39m[\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m--env=\u001b[39m\u001b[38;5;132;01m{\u001b[39;00menv\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m--num_workers=8\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m--num_envs_per_worker=4\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m--train_for_env_steps=4000000\u001b[39m\u001b[38;5;124m\"\u001b[39m])\n\u001b[0;32m----> 9\u001b[0m status \u001b[38;5;241m=\u001b[39m \u001b[43mrun_rl\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcfg\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/Desktop/course-deepRL-huggingface/unit8-part2-doom/.conda/lib/python3.10/site-packages/sample_factory/train.py:37\u001b[0m, in \u001b[0;36mrun_rl\u001b[0;34m(cfg)\u001b[0m\n\u001b[1;32m 32\u001b[0m cfg, runner \u001b[38;5;241m=\u001b[39m make_runner(cfg)\n\u001b[1;32m 34\u001b[0m \u001b[38;5;66;03m# here we can register additional message or summary handlers\u001b[39;00m\n\u001b[1;32m 35\u001b[0m \u001b[38;5;66;03m# see sf_examples/dmlab/train_dmlab.py for example\u001b[39;00m\n\u001b[0;32m---> 37\u001b[0m status \u001b[38;5;241m=\u001b[39m \u001b[43mrunner\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minit\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 38\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m status \u001b[38;5;241m==\u001b[39m ExperimentStatus\u001b[38;5;241m.\u001b[39mSUCCESS:\n\u001b[1;32m 39\u001b[0m status \u001b[38;5;241m=\u001b[39m runner\u001b[38;5;241m.\u001b[39mrun()\n", - "File \u001b[0;32m~/Desktop/course-deepRL-huggingface/unit8-part2-doom/.conda/lib/python3.10/site-packages/sample_factory/algo/runners/runner_parallel.py:43\u001b[0m, in \u001b[0;36mParallelRunner.init\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 40\u001b[0m learner_proc\u001b[38;5;241m.\u001b[39mevent_loop\u001b[38;5;241m.\u001b[39mowner \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlearners[policy_id]\n\u001b[1;32m 41\u001b[0m learner_proc\u001b[38;5;241m.\u001b[39mset_init_func_args((sf_global_context(), \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlearners[policy_id]))\n\u001b[0;32m---> 43\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msampler \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_make_sampler\u001b[49m\u001b[43m(\u001b[49m\u001b[43mParallelSampler\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mevent_loop\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 45\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconnect_components()\n\u001b[1;32m 46\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m status\n", - "File \u001b[0;32m~/Desktop/course-deepRL-huggingface/unit8-part2-doom/.conda/lib/python3.10/site-packages/sample_factory/algo/runners/runner.py:538\u001b[0m, in \u001b[0;36mRunner._make_sampler\u001b[0;34m(self, sampler_cls, event_loop)\u001b[0m\n\u001b[1;32m 536\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlearners) \u001b[38;5;241m==\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcfg\u001b[38;5;241m.\u001b[39mnum_policies, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mLearners not created yet\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 537\u001b[0m param_servers \u001b[38;5;241m=\u001b[39m {policy: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlearners[policy]\u001b[38;5;241m.\u001b[39mparam_server \u001b[38;5;28;01mfor\u001b[39;00m policy \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlearners}\n\u001b[0;32m--> 538\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43msampler_cls\u001b[49m\u001b[43m(\u001b[49m\u001b[43mevent_loop\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbuffer_mgr\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mparam_servers\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcfg\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43menv_info\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/Desktop/course-deepRL-huggingface/unit8-part2-doom/.conda/lib/python3.10/site-packages/sample_factory/algo/sampling/sampler.py:262\u001b[0m, in \u001b[0;36mParallelSampler.__init__\u001b[0;34m(self, event_loop, buffer_mgr, param_servers, cfg, env_info)\u001b[0m\n\u001b[1;32m 258\u001b[0m inference_proc \u001b[38;5;241m=\u001b[39m EventLoopProcess(\n\u001b[1;32m 259\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124minference_proc\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mpolicy_id\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m-\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mi\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m, mp_ctx, init_func\u001b[38;5;241m=\u001b[39minit_inference_process\n\u001b[1;32m 260\u001b[0m )\n\u001b[1;32m 261\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocesses\u001b[38;5;241m.\u001b[39mappend(inference_proc)\n\u001b[0;32m--> 262\u001b[0m inference_worker \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_make_inference_worker\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 263\u001b[0m \u001b[43m \u001b[49m\u001b[43minference_proc\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mevent_loop\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 264\u001b[0m \u001b[43m \u001b[49m\u001b[43mpolicy_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 265\u001b[0m \u001b[43m \u001b[49m\u001b[43mi\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 266\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpolicy_param_server\u001b[49m\u001b[43m[\u001b[49m\u001b[43mpolicy_id\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 267\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 268\u001b[0m inference_proc\u001b[38;5;241m.\u001b[39mevent_loop\u001b[38;5;241m.\u001b[39mowner \u001b[38;5;241m=\u001b[39m inference_worker\n\u001b[1;32m 269\u001b[0m inference_proc\u001b[38;5;241m.\u001b[39mset_init_func_args((sf_global_context(), inference_worker))\n", - "File \u001b[0;32m~/Desktop/course-deepRL-huggingface/unit8-part2-doom/.conda/lib/python3.10/site-packages/sample_factory/algo/sampling/sampler.py:109\u001b[0m, in \u001b[0;36mSampler._make_inference_worker\u001b[0;34m(self, event_loop, policy_id, worker_idx, param_server)\u001b[0m\n\u001b[1;32m 108\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21m_make_inference_worker\u001b[39m(\u001b[38;5;28mself\u001b[39m, event_loop, policy_id: PolicyID, worker_idx: \u001b[38;5;28mint\u001b[39m, param_server: ParameterServer):\n\u001b[0;32m--> 109\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mInferenceWorker\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 110\u001b[0m \u001b[43m \u001b[49m\u001b[43mevent_loop\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 111\u001b[0m \u001b[43m \u001b[49m\u001b[43mpolicy_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 112\u001b[0m \u001b[43m \u001b[49m\u001b[43mworker_idx\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 113\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbuffer_mgr\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 114\u001b[0m \u001b[43m \u001b[49m\u001b[43mparam_server\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 115\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minference_queues\u001b[49m\u001b[43m[\u001b[49m\u001b[43mpolicy_id\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 116\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcfg\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 117\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43menv_info\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 118\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/Desktop/course-deepRL-huggingface/unit8-part2-doom/.conda/lib/python3.10/site-packages/sample_factory/algo/sampling/inference_worker.py:93\u001b[0m, in \u001b[0;36mInferenceWorker.__init__\u001b[0;34m(self, event_loop, policy_id, worker_idx, buffer_mgr, param_server, inference_queue, cfg, env_info)\u001b[0m\n\u001b[1;32m 90\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtraj_tensors: Dict[Device, TensorDict] \u001b[38;5;241m=\u001b[39m copy\u001b[38;5;241m.\u001b[39mcopy(buffer_mgr\u001b[38;5;241m.\u001b[39mtraj_tensors_torch)\n\u001b[1;32m 91\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpolicy_output_tensors: Dict[Device, TensorDict] \u001b[38;5;241m=\u001b[39m copy\u001b[38;5;241m.\u001b[39mcopy(buffer_mgr\u001b[38;5;241m.\u001b[39mpolicy_output_tensors_torch)\n\u001b[0;32m---> 93\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdevice: torch\u001b[38;5;241m.\u001b[39mdevice \u001b[38;5;241m=\u001b[39m \u001b[43mpolicy_device\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcfg\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpolicy_id\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 94\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mparam_client \u001b[38;5;241m=\u001b[39m make_parameter_client(cfg\u001b[38;5;241m.\u001b[39mserial_mode, param_server, cfg, env_info, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtiming)\n\u001b[1;32m 95\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minference_queue \u001b[38;5;241m=\u001b[39m inference_queue\n", - "File \u001b[0;32m~/Desktop/course-deepRL-huggingface/unit8-part2-doom/.conda/lib/python3.10/site-packages/sample_factory/algo/utils/shared_buffers.py:32\u001b[0m, in \u001b[0;36mpolicy_device\u001b[0;34m(cfg, policy_id)\u001b[0m\n\u001b[1;32m 30\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mdevice(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcpu\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 31\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m---> 32\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mdevice(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcuda\u001b[39m\u001b[38;5;124m\"\u001b[39m, index\u001b[38;5;241m=\u001b[39m\u001b[43mgpus_for_process\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpolicy_id\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m)\n", - "\u001b[0;31mIndexError\u001b[0m: list index out of range" + "An exception has occurred, use %tb to see the full traceback.\n", + "\u001b[0;31mSystemExit\u001b[0m\u001b[0;31m:\u001b[0m 2\n" ] } ], @@ -659,7 +600,7 @@ "# The scenario we train on today is health gathering\n", "# other scenarios include \"doom_basic\", \"doom_two_colors_easy\", \"doom_dm\", \"doom_dwango5\", \"doom_my_way_home\", \"doom_deadly_corridor\", \"doom_defend_the_center\", \"doom_defend_the_line\"\n", "env = \"doom_health_gathering_supreme\"\n", - "cfg = parse_vizdoom_cfg(argv=[f\"--env={env}\", \"--num_workers=8\", \"--num_envs_per_worker=4\", \"--train_for_env_steps=4000000\"])\n", + "cfg = parse_vizdoom_cfg(argv=[f\"--env={env}\", \"--num_workers=6\",\"--device=cpu\" ,\"--num_envs_per_worker=2\", \"--train_for_env_steps=4000000\"])\n", "\n", "status = run_rl(cfg)" ]