Upload iter_0079

.gitattributes

@@ -48,3 +48,4 @@ iter_0059/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 iter_0064/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 iter_0069/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 iter_0074/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+iter_0079/tokenizer.json filter=lfs diff=lfs merge=lfs -text

iter_0079/README.md

@@ -0,0 +1,207 @@
+---
+base_model: Qwen/Qwen2.5-Coder-7B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen2.5-Coder-7B-Instruct
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

iter_0079/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-Coder-7B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "v_proj",
+    "k_proj",
+    "o_proj",
+    "up_proj",
+    "down_proj",
+    "gate_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

iter_0079/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b3225c2e62c5a9495cc9fe8a7205735b4fa4eacebb1afd442a458650ac4bc15
+size 161533192

iter_0079/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

iter_0079/dual_variables.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d4834e9a2da1dc706ac5112e7fb16a34bd6821b3ed492f9a4262e7a14855713
+size 1020

iter_0079/log.json

@@ -0,0 +1,1442 @@
+[
+  {
+    "iteration": 0,
+    "elapsed_s": 357.1625916957855,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7050087932900435,
+    "avg_questions": 0.75,
+    "avg_turns": 1.71875,
+    "ppo_loss": 0.007755006247732256,
+    "value_loss": 1.0904684109347207,
+    "approx_kl": 0.00942303503896775,
+    "clip_frac": 0.13966836780309677,
+    "entropy": 0.26119276721562656,
+    "kl_penalty": 0.012391226632254464,
+    "kl_per_seq": 0.012391226632254464,
+    "kl_seq_max": 0.16796875
+  },
+  {
+    "iteration": 1,
+    "elapsed_s": 343.5082986354828,
+    "lambda1": 0.009375000000000001,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.739140546953047,
+    "avg_questions": 0.84375,
+    "avg_turns": 1.84375,
+    "ppo_loss": -0.0010012539569288492,
+    "value_loss": 0.285459230421111,
+    "approx_kl": 0.006104676685936283,
+    "clip_frac": 0.08984375,
+    "entropy": 0.23781040566973388,
+    "kl_penalty": 0.011135101318359375,
+    "kl_per_seq": 0.011135101318359375,
+    "kl_seq_max": 0.136962890625
+  },
+  {
+    "iteration": 2,
+    "elapsed_s": 394.53891587257385,
+    "lambda1": 0.0125,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7890625,
+    "avg_questions": 0.78125,
+    "avg_turns": 1.78125,
+    "ppo_loss": 0.03891922673210502,
+    "value_loss": 0.637173940660432,
+    "approx_kl": 0.007546657929196954,
+    "clip_frac": 0.1015625,
+    "entropy": 0.24309116404037923,
+    "kl_penalty": 0.01812744140625,
+    "kl_per_seq": 0.01812744140625,
+    "kl_seq_max": 0.14404296875
+  },
+  {
+    "iteration": 3,
+    "elapsed_s": 300.50555062294006,
+    "lambda1": 0.0125,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7289949633699634,
+    "avg_questions": 0.75,
+    "avg_turns": 1.75,
+    "ppo_loss": -0.00402364440794502,
+    "value_loss": 0.3008941656776837,
+    "approx_kl": 0.0077281083379473,
+    "clip_frac": 0.12053571428571429,
+    "entropy": 0.23592129349708557,
+    "kl_penalty": 0.012259347098214286,
+    "kl_per_seq": 0.012259347098214286,
+    "kl_seq_max": 0.16559709821428573
+  },
+  {
+    "iteration": 4,
+    "elapsed_s": 367.04741644859314,
+    "lambda1": 0.00625,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.647762523934399,
+    "avg_questions": 0.6875,
+    "avg_turns": 1.6875,
+    "ppo_loss": 0.006391565016071711,
+    "value_loss": 0.4582739163722311,
+    "approx_kl": 0.010383476286993496,
+    "clip_frac": 0.13095238112977572,
+    "entropy": 0.2528929031853165,
+    "kl_penalty": 0.018035888671875,
+    "kl_per_seq": 0.018035888671875,
+    "kl_seq_max": 0.21428571428571427
+  },
+  {
+    "iteration": 5,
+    "elapsed_s": 382.89120960235596,
+    "lambda1": 0.00625,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.754916351010101,
+    "avg_questions": 0.75,
+    "avg_turns": 1.75,
+    "ppo_loss": 0.007902485584574086,
+    "value_loss": 0.31790804117918015,
+    "approx_kl": 0.005824520651783262,
+    "clip_frac": 0.07142857142857142,
+    "entropy": 0.25753957751606193,
+    "kl_penalty": 0.012311662946428572,
+    "kl_per_seq": 0.012311662946428572,
+    "kl_seq_max": 0.16071428571428573
+  },
+  {
+    "iteration": 6,
+    "elapsed_s": 291.7231960296631,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7139756944444444,
+    "avg_questions": 0.6875,
+    "avg_turns": 1.6875,
+    "ppo_loss": 0.01009868954618772,
+    "value_loss": 0.32571624716122943,
+    "approx_kl": 0.012746187547842661,
+    "clip_frac": 0.125,
+    "entropy": 0.24180377274751663,
+    "kl_penalty": 0.01416015625,
+    "kl_per_seq": 0.01416015625,
+    "kl_seq_max": 0.22916666666666666
+  },
+  {
+    "iteration": 7,
+    "elapsed_s": 340.50909781455994,
+    "lambda1": 0.00625,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.6727949134199136,
+    "avg_questions": 0.8125,
+    "avg_turns": 1.8125,
+    "ppo_loss": 0.007321253418922424,
+    "value_loss": 0.36242500762455165,
+    "approx_kl": 0.007387942634522915,
+    "clip_frac": 0.12890625,
+    "entropy": 0.27400176017545164,
+    "kl_penalty": 0.01043701171875,
+    "kl_per_seq": 0.01043701171875,
+    "kl_seq_max": 0.13134765625
+  },
+  {
+    "iteration": 8,
+    "elapsed_s": 330.3607187271118,
+    "lambda1": 0.0125,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.6581473214285714,
+    "avg_questions": 0.8125,
+    "avg_turns": 1.8125,
+    "ppo_loss": -0.0054003591649234295,
+    "value_loss": 0.37248116498813033,
+    "approx_kl": 0.007915348513051867,
+    "clip_frac": 0.13671875,
+    "entropy": 0.2505741387140006,
+    "kl_penalty": 0.0040740966796875,
+    "kl_per_seq": 0.0040740966796875,
+    "kl_seq_max": 0.1396484375
+  },
+  {
+    "iteration": 9,
+    "elapsed_s": 398.63353538513184,
+    "lambda1": 0.04375,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.6826368943556443,
+    "avg_questions": 1.0625,
+    "avg_turns": 1.90625,
+    "ppo_loss": -0.021302172332070768,
+    "value_loss": 1.4880596799775958,
+    "approx_kl": 0.010055532649857923,
+    "clip_frac": 0.15468750055879354,
+    "entropy": 0.2681027443613857,
+    "kl_penalty": 0.007171630859375,
+    "kl_per_seq": 0.007171630859375,
+    "kl_seq_max": 0.16552734375
+  },
+  {
+    "iteration": 10,
+    "elapsed_s": 333.02629804611206,
+    "lambda1": 0.049999999999999996,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.707235863095238,
+    "avg_questions": 0.8125,
+    "avg_turns": 1.78125,
+    "ppo_loss": -0.013005324872210622,
+    "value_loss": 0.39788789255544543,
+    "approx_kl": 0.006880114786326885,
+    "clip_frac": 0.12109375,
+    "entropy": 0.22961895586922765,
+    "kl_penalty": 3.0517578125e-05,
+    "kl_per_seq": 3.0517578125e-05,
+    "kl_seq_max": 0.128173828125
+  },
+  {
+    "iteration": 11,
+    "elapsed_s": 294.7535834312439,
+    "lambda1": 0.031249999999999993,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.8168402777777777,
+    "avg_questions": 0.5625,
+    "avg_turns": 1.5625,
+    "ppo_loss": -0.022718935273587704,
+    "value_loss": 0.3321453183889389,
+    "approx_kl": 0.006399596642170634,
+    "clip_frac": 0.10267857142857142,
+    "entropy": 0.18069506117275783,
+    "kl_penalty": 0.007045200892857143,
+    "kl_per_seq": 0.007045200892857143,
+    "kl_seq_max": 0.10574776785714286
+  },
+  {
+    "iteration": 12,
+    "elapsed_s": 343.0087254047394,
+    "lambda1": 0.03749999999999999,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.8053819444444443,
+    "avg_questions": 0.8125,
+    "avg_turns": 1.78125,
+    "ppo_loss": -0.020216253586113453,
+    "value_loss": 0.27846826042514294,
+    "approx_kl": 0.009821648010984063,
+    "clip_frac": 0.16796875,
+    "entropy": 0.25039526051841676,
+    "kl_penalty": 0.013916015625,
+    "kl_per_seq": 0.013916015625,
+    "kl_seq_max": 0.1962890625
+  },
+  {
+    "iteration": 13,
+    "elapsed_s": 363.77874422073364,
+    "lambda1": 0.05312499999999999,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.5087808892496393,
+    "avg_questions": 0.90625,
+    "avg_turns": 1.875,
+    "ppo_loss": -0.04679604054049209,
+    "value_loss": 0.3308154027219172,
+    "approx_kl": 0.013959913066139928,
+    "clip_frac": 0.16666666666666666,
+    "entropy": 0.2726931580238872,
+    "kl_penalty": 0.010181568287037037,
+    "kl_per_seq": 0.010181568287037037,
+    "kl_seq_max": 0.3732638888888889
+  },
+  {
+    "iteration": 14,
+    "elapsed_s": 352.3718099594116,
+    "lambda1": 0.05937499999999999,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.6545655733155732,
+    "avg_questions": 0.8125,
+    "avg_turns": 1.8125,
+    "ppo_loss": -0.03569141589105129,
+    "value_loss": 0.3559884512796998,
+    "approx_kl": 0.01543659484013915,
+    "clip_frac": 0.21875,
+    "entropy": 0.2455106396228075,
+    "kl_penalty": 0.01318359375,
+    "kl_per_seq": 0.01318359375,
+    "kl_seq_max": 0.3974609375
+  },
+  {
+    "iteration": 15,
+    "elapsed_s": 313.32435488700867,
+    "lambda1": 0.05312499999999999,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7722470238095239,
+    "avg_questions": 0.6875,
+    "avg_turns": 1.65625,
+    "ppo_loss": -0.0331129701091693,
+    "value_loss": 0.407362831899753,
+    "approx_kl": 0.014424336071197804,
+    "clip_frac": 0.18173076957464218,
+    "entropy": 0.23549939606052178,
+    "kl_penalty": 0.022601787860576924,
+    "kl_per_seq": 0.022601787860576924,
+    "kl_seq_max": 0.5504807692307693
+  },
+  {
+    "iteration": 16,
+    "elapsed_s": 276.3943181037903,
+    "lambda1": 0.06562499999999999,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.8410790598290598,
+    "avg_questions": 0.875,
+    "avg_turns": 1.84375,
+    "ppo_loss": -0.037679088301956654,
+    "value_loss": 0.20089253911282867,
+    "approx_kl": 0.011464653187431395,
+    "clip_frac": 0.1510416679084301,
+    "entropy": 0.20766122685745358,
+    "kl_penalty": 0.06768798828125,
+    "kl_per_seq": 0.06768798828125,
+    "kl_seq_max": 0.5673828125
+  },
+  {
+    "iteration": 17,
+    "elapsed_s": 375.74297308921814,
+    "lambda1": 0.0875,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.8539749313186813,
+    "avg_questions": 0.96875,
+    "avg_turns": 1.90625,
+    "ppo_loss": -0.0474210566530625,
+    "value_loss": 0.2925632045293848,
+    "approx_kl": 0.017900955832252898,
+    "clip_frac": 0.28083333373069763,
+    "entropy": 0.23634420384963353,
+    "kl_penalty": 0.009114583333333334,
+    "kl_per_seq": 0.009114583333333334,
+    "kl_seq_max": 0.5213541666666667
+  },
+  {
+    "iteration": 18,
+    "elapsed_s": 330.67957067489624,
+    "lambda1": 0.10625,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7826446123321124,
+    "avg_questions": 0.9375,
+    "avg_turns": 1.90625,
+    "ppo_loss": -0.055612206557079366,
+    "value_loss": 0.31383695316157845,
+    "approx_kl": 0.016603972174619373,
+    "clip_frac": 0.24210526362845772,
+    "entropy": 0.2367225446199116,
+    "kl_penalty": 0.05126953125,
+    "kl_per_seq": 0.05126953125,
+    "kl_seq_max": 0.6118421052631579
+  },
+  {
+    "iteration": 19,
+    "elapsed_s": 316.73736214637756,
+    "lambda1": 0.10625,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7678131764069264,
+    "avg_questions": 0.75,
+    "avg_turns": 1.71875,
+    "ppo_loss": -0.05081604502123335,
+    "value_loss": 0.3012801162574602,
+    "approx_kl": 0.015763986157253385,
+    "clip_frac": 0.16459627384724823,
+    "entropy": 0.21080296000708704,
+    "kl_penalty": 0.10081946331521739,
+    "kl_per_seq": 0.10081946331521739,
+    "kl_seq_max": 0.7853260869565217
+  },
+  {
+    "iteration": 20,
+    "elapsed_s": 292.9455828666687,
+    "lambda1": 0.1125,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7923935439560439,
+    "avg_questions": 0.8125,
+    "avg_turns": 1.75,
+    "ppo_loss": -0.03436802277962367,
+    "value_loss": 0.327542544901371,
+    "approx_kl": 0.016658279796441397,
+    "clip_frac": 0.2,
+    "entropy": 0.2346005121866862,
+    "kl_penalty": 0.07431640625,
+    "kl_per_seq": 0.07431640625,
+    "kl_seq_max": 0.7651041666666667
+  },
+  {
+    "iteration": 21,
+    "elapsed_s": 316.87915897369385,
+    "lambda1": 0.10625,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7417455808080807,
+    "avg_questions": 0.6875,
+    "avg_turns": 1.6875,
+    "ppo_loss": -0.045961443748739034,
+    "value_loss": 0.4759538339243995,
+    "approx_kl": 0.015832876993550196,
+    "clip_frac": 0.22685185240374672,
+    "entropy": 0.2288952949974272,
+    "kl_penalty": 0.08394368489583333,
+    "kl_per_seq": 0.08394368489583333,
+    "kl_seq_max": 0.7795138888888888
+  },
+  {
+    "iteration": 22,
+    "elapsed_s": 305.55787348747253,
+    "lambda1": 0.09999999999999999,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.8609916125541125,
+    "avg_questions": 0.6875,
+    "avg_turns": 1.6875,
+    "ppo_loss": -0.05956608028366016,
+    "value_loss": 0.24114569362539512,
+    "approx_kl": 0.019050200308601443,
+    "clip_frac": 0.24839743685263854,
+    "entropy": 0.21138634217473176,
+    "kl_penalty": 0.06511981670673077,
+    "kl_per_seq": 0.06511981670673077,
+    "kl_seq_max": 0.8031850961538461
+  },
+  {
+    "iteration": 23,
+    "elapsed_s": 321.8149151802063,
+    "lambda1": 0.109375,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7557539682539682,
+    "avg_questions": 0.84375,
+    "avg_turns": 1.8125,
+    "ppo_loss": -0.056632211245596406,
+    "value_loss": 0.3020759107545018,
+    "approx_kl": 0.01828147955238819,
+    "clip_frac": 0.3,
+    "entropy": 0.24711562618613242,
+    "kl_penalty": 0.101806640625,
+    "kl_per_seq": 0.101806640625,
+    "kl_seq_max": 0.84375
+  },
+  {
+    "iteration": 24,
+    "elapsed_s": 306.5719954967499,
+    "lambda1": 0.109375,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7853937728937729,
+    "avg_questions": 0.75,
+    "avg_turns": 1.75,
+    "ppo_loss": -0.037689958016077676,
+    "value_loss": 0.39611613551775615,
+    "approx_kl": 0.01130907932917277,
+    "clip_frac": 0.15,
+    "entropy": 0.1994072715441386,
+    "kl_penalty": 0.18115234375,
+    "kl_per_seq": 0.18115234375,
+    "kl_seq_max": 1.1114583333333334
+  },
+  {
+    "iteration": 25,
+    "elapsed_s": 328.52625131607056,
+    "lambda1": 0.103125,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.6865439248251748,
+    "avg_questions": 0.6875,
+    "avg_turns": 1.65625,
+    "ppo_loss": -0.04527500409044718,
+    "value_loss": 0.47288530751278524,
+    "approx_kl": 0.015802607822575067,
+    "clip_frac": 0.19868421084002444,
+    "entropy": 0.20268321821564123,
+    "kl_penalty": 0.16164679276315788,
+    "kl_per_seq": 0.16164679276315788,
+    "kl_seq_max": 1.0608552631578947
+  },
+  {
+    "iteration": 26,
+    "elapsed_s": 301.34392499923706,
+    "lambda1": 0.10625,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7395833333333334,
+    "avg_questions": 0.78125,
+    "avg_turns": 1.6875,
+    "ppo_loss": -0.04237345000728965,
+    "value_loss": 0.4161538938060403,
+    "approx_kl": 0.015270014468114823,
+    "clip_frac": 0.15625000093132257,
+    "entropy": 0.1977705955505371,
+    "kl_penalty": 0.2110595703125,
+    "kl_per_seq": 0.2110595703125,
+    "kl_seq_max": 1.24658203125
+  },
+  {
+    "iteration": 27,
+    "elapsed_s": 348.94201493263245,
+    "lambda1": 0.1125,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7194080225330226,
+    "avg_questions": 0.8125,
+    "avg_turns": 1.8125,
+    "ppo_loss": -0.05180658517699493,
+    "value_loss": 0.2778499259760505,
+    "approx_kl": 0.01625884206671464,
+    "clip_frac": 0.19736842105263158,
+    "entropy": 0.21087203449324557,
+    "kl_penalty": 0.13524105674342105,
+    "kl_per_seq": 0.13524105674342105,
+    "kl_seq_max": 1.1488486842105263
+  },
+  {
+    "iteration": 28,
+    "elapsed_s": 322.038512468338,
+    "lambda1": 0.109375,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.6542670003607503,
+    "avg_questions": 0.71875,
+    "avg_turns": 1.65625,
+    "ppo_loss": -0.04524578066433177,
+    "value_loss": 0.4445836964775534,
+    "approx_kl": 0.019655843183179113,
+    "clip_frac": 0.21176470640827627,
+    "entropy": 0.21890325931941762,
+    "kl_penalty": 0.08786190257352941,
+    "kl_per_seq": 0.08786190257352941,
+    "kl_seq_max": 0.859375
+  },
+  {
+    "iteration": 29,
+    "elapsed_s": 339.1300230026245,
+    "lambda1": 0.096875,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7828609758297258,
+    "avg_questions": 0.625,
+    "avg_turns": 1.625,
+    "ppo_loss": -0.059186975226590506,
+    "value_loss": 0.31758245355204534,
+    "approx_kl": 0.018988314819963353,
+    "clip_frac": 0.23684210526315788,
+    "entropy": 0.20613071636149757,
+    "kl_penalty": 0.34781044407894735,
+    "kl_per_seq": 0.34781044407894735,
+    "kl_seq_max": 1.433388157894737
+  },
+  {
+    "iteration": 30,
+    "elapsed_s": 290.3177366256714,
+    "lambda1": 0.084375,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.684590583028083,
+    "avg_questions": 0.625,
+    "avg_turns": 1.625,
+    "ppo_loss": -0.039527233276102275,
+    "value_loss": 0.35584185810552704,
+    "approx_kl": 0.021562429351939097,
+    "clip_frac": 0.2777777777777778,
+    "entropy": 0.23942414422829947,
+    "kl_penalty": 0.15489366319444445,
+    "kl_per_seq": 0.15489366319444445,
+    "kl_seq_max": 1.1883680555555556
+  },
+  {
+    "iteration": 31,
+    "elapsed_s": 299.3714759349823,
+    "lambda1": 0.084375,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.875,
+    "avg_questions": 0.75,
+    "avg_turns": 1.6875,
+    "ppo_loss": -0.044965135399252176,
+    "value_loss": 0.2800211445428431,
+    "approx_kl": 0.018849731306545436,
+    "clip_frac": 0.1875,
+    "entropy": 0.2140436777845025,
+    "kl_penalty": 0.294921875,
+    "kl_per_seq": 0.294921875,
+    "kl_seq_max": 1.630859375
+  },
+  {
+    "iteration": 32,
+    "elapsed_s": 351.7063624858856,
+    "lambda1": 0.078125,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7589318667443667,
+    "avg_questions": 0.6875,
+    "avg_turns": 1.6875,
+    "ppo_loss": -0.03741050139069557,
+    "value_loss": 0.28676399641803335,
+    "approx_kl": 0.023076566202299937,
+    "clip_frac": 0.24702381023338862,
+    "entropy": 0.21046061547739164,
+    "kl_penalty": 0.3627232142857143,
+    "kl_per_seq": 0.3627232142857143,
+    "kl_seq_max": 1.9888392857142858
+  },
+  {
+    "iteration": 33,
+    "elapsed_s": 402.1016628742218,
+    "lambda1": 0.075,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.827605294011544,
+    "avg_questions": 0.71875,
+    "avg_turns": 1.71875,
+    "ppo_loss": -0.040155550331941674,
+    "value_loss": 0.20966205213751113,
+    "approx_kl": 0.017562197853944132,
+    "clip_frac": 0.21811224626643316,
+    "entropy": 0.21647065665040696,
+    "kl_penalty": 0.28662109375,
+    "kl_per_seq": 0.28662109375,
+    "kl_seq_max": 2.1021205357142856
+  },
+  {
+    "iteration": 34,
+    "elapsed_s": 324.4843943119049,
+    "lambda1": 0.08125,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7170138888888888,
+    "avg_questions": 0.8125,
+    "avg_turns": 1.75,
+    "ppo_loss": -0.030176887458020992,
+    "value_loss": 0.3569460050626235,
+    "approx_kl": 0.019110250879417767,
+    "clip_frac": 0.22727272727272727,
+    "entropy": 0.20825903794982217,
+    "kl_penalty": 0.5365767045454546,
+    "kl_per_seq": 0.5365767045454546,
+    "kl_seq_max": 2.335227272727273
+  },
+  {
+    "iteration": 35,
+    "elapsed_s": 315.85867977142334,
+    "lambda1": 0.06875,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7331473214285715,
+    "avg_questions": 0.625,
+    "avg_turns": 1.53125,
+    "ppo_loss": -0.006593746634630056,
+    "value_loss": 0.4972928464412689,
+    "approx_kl": 0.015867754243887387,
+    "clip_frac": 0.19230769230769232,
+    "entropy": 0.23934545998389906,
+    "kl_penalty": 0.43111478365384615,
+    "kl_per_seq": 0.43111478365384615,
+    "kl_seq_max": 2.6298076923076925
+  },
+  {
+    "iteration": 36,
+    "elapsed_s": 386.9313094615936,
+    "lambda1": 0.084375,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.8004261363636361,
+    "avg_questions": 0.90625,
+    "avg_turns": 1.84375,
+    "ppo_loss": 0.003060997887091203,
+    "value_loss": 0.30295307460156357,
+    "approx_kl": 0.01550799777561968,
+    "clip_frac": 0.2121212130243128,
+    "entropy": 0.22590754248879172,
+    "kl_penalty": 0.33540482954545453,
+    "kl_per_seq": 0.33540482954545453,
+    "kl_seq_max": 2.1988636363636362
+  },
+  {
+    "iteration": 37,
+    "elapsed_s": 372.7559549808502,
+    "lambda1": 0.078125,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7559456168831168,
+    "avg_questions": 0.6875,
+    "avg_turns": 1.65625,
+    "ppo_loss": -0.041432250601549946,
+    "value_loss": 0.3984727369000514,
+    "approx_kl": 0.024890107413132984,
+    "clip_frac": 0.2562500002483527,
+    "entropy": 0.22201429742077985,
+    "kl_penalty": 0.7021484375,
+    "kl_per_seq": 0.7021484375,
+    "kl_seq_max": 3.1744791666666665
+  },
+  {
+    "iteration": 38,
+    "elapsed_s": 349.12115454673767,
+    "lambda1": 0.06875,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7446377840909091,
+    "avg_questions": 0.65625,
+    "avg_turns": 1.65625,
+    "ppo_loss": -0.011596236005425453,
+    "value_loss": 0.3541109532117844,
+    "approx_kl": 0.01865895972587168,
+    "clip_frac": 0.175,
+    "entropy": 0.21150051951408386,
+    "kl_penalty": 0.7384765625,
+    "kl_per_seq": 0.7384765625,
+    "kl_seq_max": 3.715625
+  },
+  {
+    "iteration": 39,
+    "elapsed_s": 405.69459891319275,
+    "lambda1": 0.05,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7032117361804863,
+    "avg_questions": 0.5625,
+    "avg_turns": 1.5625,
+    "ppo_loss": 0.028779844088213786,
+    "value_loss": 0.3979421172823225,
+    "approx_kl": 0.018495287214006697,
+    "clip_frac": 0.21428571428571427,
+    "entropy": 0.18224240839481354,
+    "kl_penalty": 1.3233816964285714,
+    "kl_per_seq": 1.3233816964285714,
+    "kl_seq_max": 3.21875
+  },
+  {
+    "iteration": 40,
+    "elapsed_s": 477.3596832752228,
+    "lambda1": 0.037500000000000006,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7178388451825952,
+    "avg_questions": 0.625,
+    "avg_turns": 1.5625,
+    "ppo_loss": -0.053192346141888544,
+    "value_loss": 0.4054635831942925,
+    "approx_kl": 0.019211487128184393,
+    "clip_frac": 0.28846153846153844,
+    "entropy": 0.1979089849270307,
+    "kl_penalty": 1.4969951923076923,
+    "kl_per_seq": 1.4969951923076923,
+    "kl_seq_max": 4.668269230769231
+  },
+  {
+    "iteration": 41,
+    "elapsed_s": 529.4953544139862,
+    "lambda1": 0.0062500000000000056,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.8039234203296702,
+    "avg_questions": 0.4375,
+    "avg_turns": 1.4375,
+    "ppo_loss": -0.010720850278933844,
+    "value_loss": 0.440558356543382,
+    "approx_kl": 0.015522866044193506,
+    "clip_frac": 0.20138889054457346,
+    "entropy": 0.15809335187077522,
+    "kl_penalty": 1.9837239583333333,
+    "kl_per_seq": 1.9837239583333333,
+    "kl_seq_max": 5.434895833333333
+  },
+  {
+    "iteration": 42,
+    "elapsed_s": 399.98438239097595,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7863238324175824,
+    "avg_questions": 0.59375,
+    "avg_turns": 1.5625,
+    "ppo_loss": -0.019172271092732746,
+    "value_loss": 0.38196369177765316,
+    "approx_kl": 0.015519324276182387,
+    "clip_frac": 0.2638888888888889,
+    "entropy": 0.20362108945846558,
+    "kl_penalty": 1.5021701388888888,
+    "kl_per_seq": 1.5021701388888888,
+    "kl_seq_max": 3.8177083333333335
+  },
+  {
+    "iteration": 43,
+    "elapsed_s": 382.99004435539246,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7528521825396826,
+    "avg_questions": 0.625,
+    "avg_turns": 1.59375,
+    "ppo_loss": -0.04032935032790357,
+    "value_loss": 0.3547650304707614,
+    "approx_kl": 0.016969902102242817,
+    "clip_frac": 0.18939394029704007,
+    "entropy": 0.20878374915231357,
+    "kl_penalty": 0.7615411931818182,
+    "kl_per_seq": 0.7615411931818182,
+    "kl_seq_max": 3.0369318181818183
+  },
+  {
+    "iteration": 44,
+    "elapsed_s": 357.9368350505829,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7684809981684981,
+    "avg_questions": 0.6875,
+    "avg_turns": 1.53125,
+    "ppo_loss": -0.018041244623335926,
+    "value_loss": 0.4228832056661221,
+    "approx_kl": 0.014606282114982605,
+    "clip_frac": 0.125,
+    "entropy": 0.17023578387769786,
+    "kl_penalty": 1.1877663352272727,
+    "kl_per_seq": 1.1877663352272727,
+    "kl_seq_max": 4.089488636363637
+  },
+  {
+    "iteration": 45,
+    "elapsed_s": 405.43791604042053,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7991586538461538,
+    "avg_questions": 0.3125,
+    "avg_turns": 1.28125,
+    "ppo_loss": -0.045360571808285184,
+    "value_loss": 0.39695970714092255,
+    "approx_kl": 0.01792952087190416,
+    "clip_frac": 0.125,
+    "entropy": 0.13567894986934131,
+    "kl_penalty": 2.1354166666666665,
+    "kl_per_seq": 2.1354166666666665,
+    "kl_seq_max": 5.666666666666667
+  },
+  {
+    "iteration": 46,
+    "elapsed_s": 387.11958026885986,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.9118818681318681,
+    "avg_questions": 0.75,
+    "avg_turns": 1.6875,
+    "ppo_loss": -0.02657535175482432,
+    "value_loss": 0.34566326811909676,
+    "approx_kl": 0.014782948031400641,
+    "clip_frac": 0.16666666666666666,
+    "entropy": 0.20470738597214222,
+    "kl_penalty": 1.6569010416666667,
+    "kl_per_seq": 1.6569010416666667,
+    "kl_seq_max": 4.783854166666667
+  },
+  {
+    "iteration": 47,
+    "elapsed_s": 377.47320914268494,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.8429086538461539,
+    "avg_questions": 0.40625,
+    "avg_turns": 1.375,
+    "ppo_loss": -0.015141616575419903,
+    "value_loss": 0.40079813450574875,
+    "approx_kl": 0.020528918132185936,
+    "clip_frac": 0.1875,
+    "entropy": 0.15663179010152817,
+    "kl_penalty": 2.0159505208333335,
+    "kl_per_seq": 2.0159505208333335,
+    "kl_seq_max": 5.315104166666667
+  },
+  {
+    "iteration": 48,
+    "elapsed_s": 396.71141362190247,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7468449519230769,
+    "avg_questions": 0.65625,
+    "avg_turns": 1.53125,
+    "ppo_loss": -0.04271022975444794,
+    "value_loss": 0.4582360867749561,
+    "approx_kl": 0.01956469091502103,
+    "clip_frac": 0.22727272727272727,
+    "entropy": 0.16423014873130756,
+    "kl_penalty": 1.7510653409090908,
+    "kl_per_seq": 1.7510653409090908,
+    "kl_seq_max": 4.342329545454546
+  },
+  {
+    "iteration": 49,
+    "elapsed_s": 402.4066619873047,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.8324337121212121,
+    "avg_questions": 0.625,
+    "avg_turns": 1.5625,
+    "ppo_loss": 0.011342465132474899,
+    "value_loss": 0.4689230710268021,
+    "approx_kl": 0.020286041498184203,
+    "clip_frac": 0.2875,
+    "entropy": 0.19571467787027358,
+    "kl_penalty": 1.533984375,
+    "kl_per_seq": 1.533984375,
+    "kl_seq_max": 5.15
+  },
+  {
+    "iteration": 50,
+    "elapsed_s": 468.35541796684265,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7713893745143745,
+    "avg_questions": 0.625,
+    "avg_turns": 1.5,
+    "ppo_loss": -0.02598505907438018,
+    "value_loss": 0.4095265038988807,
+    "approx_kl": 0.01934722743251107,
+    "clip_frac": 0.25,
+    "entropy": 0.18899625133384357,
+    "kl_penalty": 1.6466619318181819,
+    "kl_per_seq": 1.6466619318181819,
+    "kl_seq_max": 4.2414772727272725
+  },
+  {
+    "iteration": 51,
+    "elapsed_s": 460.11231803894043,
+    "lambda1": 0.03125,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.8237798659673661,
+    "avg_questions": 1.0625,
+    "avg_turns": 1.75,
+    "ppo_loss": -0.03820386209658214,
+    "value_loss": 0.40805009113890783,
+    "approx_kl": 0.022062738026891435,
+    "clip_frac": 0.30357142857142855,
+    "entropy": 0.22303845254438265,
+    "kl_penalty": 1.9553571428571428,
+    "kl_per_seq": 1.9553571428571428,
+    "kl_seq_max": 5.939732142857143
+  },
+  {
+    "iteration": 52,
+    "elapsed_s": 362.34188413619995,
+    "lambda1": 0.012499999999999997,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7426739926739926,
+    "avg_questions": 0.5625,
+    "avg_turns": 1.5,
+    "ppo_loss": -0.01608191430568695,
+    "value_loss": 0.5749707602792316,
+    "approx_kl": 0.016227867868211534,
+    "clip_frac": 0.19444444444444445,
+    "entropy": 0.209986950788233,
+    "kl_penalty": 1.7092013888888888,
+    "kl_per_seq": 1.7092013888888888,
+    "kl_seq_max": 4.623263888888889
+  },
+  {
+    "iteration": 53,
+    "elapsed_s": 389.7703387737274,
+    "lambda1": 0.018749999999999996,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.759375,
+    "avg_questions": 0.8125,
+    "avg_turns": 1.625,
+    "ppo_loss": -0.03594827217360338,
+    "value_loss": 0.5962108758588632,
+    "approx_kl": 0.019294889643788338,
+    "clip_frac": 0.25,
+    "entropy": 0.21581400309999785,
+    "kl_penalty": 1.6305338541666667,
+    "kl_per_seq": 1.6305338541666667,
+    "kl_seq_max": 6.4375
+  },
+  {
+    "iteration": 54,
+    "elapsed_s": 364.2307140827179,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7348958333333334,
+    "avg_questions": 0.53125,
+    "avg_turns": 1.5,
+    "ppo_loss": -0.032697478478605095,
+    "value_loss": 0.39433746852658014,
+    "approx_kl": 0.020013599233193832,
+    "clip_frac": 0.2840909090909091,
+    "entropy": 0.18534629046916962,
+    "kl_penalty": 1.7136896306818181,
+    "kl_per_seq": 1.7136896306818181,
+    "kl_seq_max": 4.588068181818182
+  },
+  {
+    "iteration": 55,
+    "elapsed_s": 430.8543348312378,
+    "lambda1": 0.0125,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.8713541666666667,
+    "avg_questions": 0.875,
+    "avg_turns": 1.75,
+    "ppo_loss": -0.03695010795043065,
+    "value_loss": 0.7556464666357408,
+    "approx_kl": 0.019689600628155928,
+    "clip_frac": 0.2980769230769231,
+    "entropy": 0.22688038074053252,
+    "kl_penalty": 1.1787860576923077,
+    "kl_per_seq": 1.1787860576923077,
+    "kl_seq_max": 4.625
+  },
+  {
+    "iteration": 56,
+    "elapsed_s": 433.31743574142456,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7333306623931624,
+    "avg_questions": 0.53125,
+    "avg_turns": 1.375,
+    "ppo_loss": -0.022642335431142288,
+    "value_loss": 0.8812255778095939,
+    "approx_kl": 0.024949382651935924,
+    "clip_frac": 0.3181818181818182,
+    "entropy": 0.21958645365454935,
+    "kl_penalty": 2.0880681818181817,
+    "kl_per_seq": 2.0880681818181817,
+    "kl_seq_max": 5.860795454545454
+  },
+  {
+    "iteration": 57,
+    "elapsed_s": 275.3117332458496,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7060876623376624,
+    "avg_questions": 0.5,
+    "avg_turns": 1.4375,
+    "ppo_loss": -0.008817820676735469,
+    "value_loss": 0.46510996563094004,
+    "approx_kl": 0.01849946432879993,
+    "clip_frac": 0.20238095309053147,
+    "entropy": 0.2783463256699698,
+    "kl_penalty": 2.1830357142857144,
+    "kl_per_seq": 2.1830357142857144,
+    "kl_seq_max": 5.816964285714286
+  },
+  {
+    "iteration": 58,
+    "elapsed_s": 407.897070646286,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.5490298763736264,
+    "avg_questions": 0.625,
+    "avg_turns": 1.4375,
+    "ppo_loss": -0.04316115958823098,
+    "value_loss": 0.557997938659456,
+    "approx_kl": 0.023318295418802235,
+    "clip_frac": 0.33796296351485783,
+    "entropy": 0.22894740766949123,
+    "kl_penalty": 1.7825520833333333,
+    "kl_per_seq": 1.7825520833333333,
+    "kl_seq_max": 6.590277777777778
+  },
+  {
+    "iteration": 59,
+    "elapsed_s": 348.05943036079407,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.777881189990565,
+    "avg_questions": 0.4375,
+    "avg_turns": 1.40625,
+    "ppo_loss": -0.011562331446579524,
+    "value_loss": 0.33755460381507874,
+    "approx_kl": 0.011728399782441556,
+    "clip_frac": 0.125,
+    "entropy": 0.18932311236858368,
+    "kl_penalty": 2.549107142857143,
+    "kl_per_seq": 2.549107142857143,
+    "kl_seq_max": 7.553571428571429
+  },
+  {
+    "iteration": 60,
+    "elapsed_s": 362.03548216819763,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.6952853136446886,
+    "avg_questions": 0.6875,
+    "avg_turns": 1.5625,
+    "ppo_loss": -0.0452236607670784,
+    "value_loss": 0.6653054714202881,
+    "approx_kl": 0.021763411909341814,
+    "clip_frac": 0.2625,
+    "entropy": 0.2450494162738323,
+    "kl_penalty": 2.24765625,
+    "kl_per_seq": 2.24765625,
+    "kl_seq_max": 5.915625
+  },
+  {
+    "iteration": 61,
+    "elapsed_s": 381.31226229667664,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.8833953373015875,
+    "avg_questions": 0.65625,
+    "avg_turns": 1.5625,
+    "ppo_loss": 0.007732013861338298,
+    "value_loss": 0.3652803964085049,
+    "approx_kl": 0.01360528998904758,
+    "clip_frac": 0.1111111111111111,
+    "entropy": 0.18831726908683777,
+    "kl_penalty": 2.165798611111111,
+    "kl_per_seq": 2.165798611111111,
+    "kl_seq_max": 6.347222222222222
+  },
+  {
+    "iteration": 62,
+    "elapsed_s": 406.71424865722656,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.861770260989011,
+    "avg_questions": 0.625,
+    "avg_turns": 1.5625,
+    "ppo_loss": -0.003276779626806577,
+    "value_loss": 0.2927818534274896,
+    "approx_kl": 0.022673659026622772,
+    "clip_frac": 0.2708333333333333,
+    "entropy": 0.21082905804117522,
+    "kl_penalty": 2.328125,
+    "kl_per_seq": 2.328125,
+    "kl_seq_max": 6.286458333333333
+  },
+  {
+    "iteration": 63,
+    "elapsed_s": 340.36828660964966,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7010022095959596,
+    "avg_questions": 0.59375,
+    "avg_turns": 1.5,
+    "ppo_loss": 0.003351894517739614,
+    "value_loss": 0.5284540810518794,
+    "approx_kl": 0.020767734282546572,
+    "clip_frac": 0.25,
+    "entropy": 0.19708349804083505,
+    "kl_penalty": 1.8897569444444444,
+    "kl_per_seq": 1.8897569444444444,
+    "kl_seq_max": 6.052083333333333
+  },
+  {
+    "iteration": 64,
+    "elapsed_s": 333.7256259918213,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.8078997825091575,
+    "avg_questions": 0.53125,
+    "avg_turns": 1.4375,
+    "ppo_loss": 0.0036465544253587723,
+    "value_loss": 0.5177192717790604,
+    "approx_kl": 0.017667772877030075,
+    "clip_frac": 0.275,
+    "entropy": 0.25572284162044523,
+    "kl_penalty": 2.2875,
+    "kl_per_seq": 2.2875,
+    "kl_seq_max": 7.20625
+  },
+  {
+    "iteration": 65,
+    "elapsed_s": 416.62586760520935,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.6066592261904762,
+    "avg_questions": 0.4375,
+    "avg_turns": 1.34375,
+    "ppo_loss": 0.004297132603824139,
+    "value_loss": 0.5007202178239822,
+    "approx_kl": 0.024690295103937387,
+    "clip_frac": 0.2864583358168602,
+    "entropy": 0.24572660587728024,
+    "kl_penalty": 2.3369140625,
+    "kl_per_seq": 2.3369140625,
+    "kl_seq_max": 5.109375
+  },
+  {
+    "iteration": 66,
+    "elapsed_s": 453.08995389938354,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7335565476190475,
+    "avg_questions": 0.625,
+    "avg_turns": 1.53125,
+    "ppo_loss": -0.04715024493634701,
+    "value_loss": 0.5405951490004858,
+    "approx_kl": 0.023815928027033806,
+    "clip_frac": 0.3229166666666667,
+    "entropy": 0.21148921192313233,
+    "kl_penalty": 1.7861328125,
+    "kl_per_seq": 1.7861328125,
+    "kl_seq_max": 5.104166666666667
+  },
+  {
+    "iteration": 67,
+    "elapsed_s": 397.9906346797943,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.68397618006993,
+    "avg_questions": 0.5,
+    "avg_turns": 1.375,
+    "ppo_loss": -0.04584093723032209,
+    "value_loss": 0.5735702597432666,
+    "approx_kl": 0.023069898287455242,
+    "clip_frac": 0.2361111111111111,
+    "entropy": 0.24479632079601288,
+    "kl_penalty": 1.2495659722222223,
+    "kl_per_seq": 1.2495659722222223,
+    "kl_seq_max": 4.828125
+  },
+  {
+    "iteration": 68,
+    "elapsed_s": 515.9583985805511,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.8773008241758242,
+    "avg_questions": 0.46875,
+    "avg_turns": 1.40625,
+    "ppo_loss": 0.0036080868707762826,
+    "value_loss": 0.512065158949958,
+    "approx_kl": 0.020016038169463474,
+    "clip_frac": 0.23888888955116272,
+    "entropy": 0.17518086814218098,
+    "kl_penalty": 2.607638888888889,
+    "kl_per_seq": 2.607638888888889,
+    "kl_seq_max": 7.270833333333333
+  },
+  {
+    "iteration": 69,
+    "elapsed_s": 383.84572100639343,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.6992120726495727,
+    "avg_questions": 0.21875,
+    "avg_turns": 1.1875,
+    "ppo_loss": -0.015125544120868048,
+    "value_loss": 0.47944863471719956,
+    "approx_kl": 0.031841082498431206,
+    "clip_frac": 0.3657407412926356,
+    "entropy": 0.20614748530917698,
+    "kl_penalty": 1.5974392361111112,
+    "kl_per_seq": 1.5974392361111112,
+    "kl_seq_max": 5.649305555555555
+  },
+  {
+    "iteration": 70,
+    "elapsed_s": 411.1189477443695,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.6458848443223444,
+    "avg_questions": 0.375,
+    "avg_turns": 1.3125,
+    "ppo_loss": -0.06534885130822658,
+    "value_loss": 0.5014325216412544,
+    "approx_kl": 0.018668577820062638,
+    "clip_frac": 0.3375,
+    "entropy": 0.21221508607268333,
+    "kl_penalty": 2.590625,
+    "kl_per_seq": 2.590625,
+    "kl_seq_max": 6.6375
+  },
+  {
+    "iteration": 71,
+    "elapsed_s": 436.6554756164551,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.8079427083333334,
+    "avg_questions": 0.5,
+    "avg_turns": 1.4375,
+    "ppo_loss": -0.010480208865677318,
+    "value_loss": 0.5480635464191437,
+    "approx_kl": 0.023314454495751608,
+    "clip_frac": 0.2847222226361434,
+    "entropy": 0.20220119630297026,
+    "kl_penalty": 3.4348958333333335,
+    "kl_per_seq": 3.4348958333333335,
+    "kl_seq_max": 7.927083333333333
+  },
+  {
+    "iteration": 72,
+    "elapsed_s": 378.03028297424316,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.5371899801587303,
+    "avg_questions": 0.34375,
+    "avg_turns": 1.25,
+    "ppo_loss": -0.028591745533049107,
+    "value_loss": 0.6426007933914661,
+    "approx_kl": 0.022364872507750988,
+    "clip_frac": 0.1875,
+    "entropy": 0.21883758902549744,
+    "kl_penalty": 2.76171875,
+    "kl_per_seq": 2.76171875,
+    "kl_seq_max": 6.91015625
+  },
+  {
+    "iteration": 73,
+    "elapsed_s": 414.9528992176056,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.585453869047619,
+    "avg_questions": 0.46875,
+    "avg_turns": 1.375,
+    "ppo_loss": 0.0015036753482288783,
+    "value_loss": 0.3997256424691942,
+    "approx_kl": 0.0357725934849845,
+    "clip_frac": 0.3333333333333333,
+    "entropy": 0.22652536713414723,
+    "kl_penalty": 2.1744791666666665,
+    "kl_per_seq": 2.1744791666666665,
+    "kl_seq_max": 5.909722222222222
+  },
+  {
+    "iteration": 74,
+    "elapsed_s": 386.3267168998718,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.5911458333333334,
+    "avg_questions": 0.375,
+    "avg_turns": 1.3125,
+    "ppo_loss": 0.08960729589064916,
+    "value_loss": 0.5382433210810026,
+    "approx_kl": 0.036540587743123375,
+    "clip_frac": 0.1875,
+    "entropy": 0.21024571855862936,
+    "kl_penalty": 2.154296875,
+    "kl_per_seq": 2.154296875,
+    "kl_seq_max": 6.359375
+  },
+  {
+    "iteration": 75,
+    "elapsed_s": 329.0774829387665,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.59921875,
+    "avg_questions": 0.09375,
+    "avg_turns": 1.09375,
+    "ppo_loss": -0.020652418350800873,
+    "value_loss": 0.31655372530221937,
+    "approx_kl": 0.026534833014011383,
+    "clip_frac": 0.24166666865348815,
+    "entropy": 0.1351238675415516,
+    "kl_penalty": 1.4828125,
+    "kl_per_seq": 1.4828125,
+    "kl_seq_max": 5.84375
+  },
+  {
+    "iteration": 76,
+    "elapsed_s": 390.8202135562897,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.5967075892857142,
+    "avg_questions": 0.375,
+    "avg_turns": 1.3125,
+    "ppo_loss": -0.026518651180797152,
+    "value_loss": 0.4646948468354013,
+    "approx_kl": 0.023818027642038133,
+    "clip_frac": 0.3055555555555556,
+    "entropy": 0.18710649965537918,
+    "kl_penalty": 1.3765190972222223,
+    "kl_per_seq": 1.3765190972222223,
+    "kl_seq_max": 5.059027777777778
+  },
+  {
+    "iteration": 77,
+    "elapsed_s": 327.5475916862488,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7489583333333332,
+    "avg_questions": 0.21875,
+    "avg_turns": 1.1875,
+    "ppo_loss": -0.0022389711812138557,
+    "value_loss": 0.5141064673662186,
+    "approx_kl": 0.016470473259687424,
+    "clip_frac": 0.125,
+    "entropy": 0.17073741927742958,
+    "kl_penalty": 0.94677734375,
+    "kl_per_seq": 0.94677734375,
+    "kl_seq_max": 4.05859375
+  },
+  {
+    "iteration": 78,
+    "elapsed_s": 389.4913065433502,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7328699945887447,
+    "avg_questions": 0.3125,
+    "avg_turns": 1.25,
+    "ppo_loss": -0.03418560006788799,
+    "value_loss": 0.4716380217245647,
+    "approx_kl": 0.019095165388924733,
+    "clip_frac": 0.16071428571428573,
+    "entropy": 0.16320552676916122,
+    "kl_penalty": 1.0892857142857142,
+    "kl_per_seq": 1.0892857142857142,
+    "kl_seq_max": 5.169642857142857
+  },
+  {
+    "iteration": 79,
+    "elapsed_s": 341.183709859848,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7184895833333333,
+    "avg_questions": 0.3125,
+    "avg_turns": 1.28125,
+    "ppo_loss": -0.01578974910080433,
+    "value_loss": 0.40421794168651104,
+    "approx_kl": 0.02196552697569132,
+    "clip_frac": 0.234375,
+    "entropy": 0.17320350836962461,
+    "kl_penalty": 1.07275390625,
+    "kl_per_seq": 1.07275390625,
+    "kl_seq_max": 3.96875
+  }
+]

iter_0079/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dbb9ccc29b4517e9dccf26d1b9f1bb48113a6eb969a61c50ba9c8a2b0089d586
+size 11421990

iter_0079/tokenizer_config.json

@@ -0,0 +1,31 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "left",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "truncation_side": "left",
+  "unk_token": null
+}

iter_0079/train_state.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:816c56e063b88ba2c6f3fb2a30f0cda1873729b40e245cc474f0a08cfa12dc1d
+size 104946244

iter_0079/value_heads.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74284f9adf14bed8d8cdaa540f8d495b098d492a4d04fdc0c1200a19388f04cd
+size 44096056