Upload iter_0044

.gitattributes

@@ -41,3 +41,4 @@ iter_0024/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 iter_0029/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 iter_0034/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 iter_0039/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+iter_0044/tokenizer.json filter=lfs diff=lfs merge=lfs -text

iter_0044/README.md

@@ -0,0 +1,207 @@
+---
+base_model: Qwen/Qwen2.5-Coder-7B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen2.5-Coder-7B-Instruct
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

iter_0044/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-Coder-7B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "k_proj",
+    "o_proj",
+    "gate_proj",
+    "up_proj",
+    "down_proj",
+    "q_proj",
+    "v_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

iter_0044/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb07145b4fc0ec8335606ecaa3b0bd126f66e958b4177c11bf0b1a202d0e90f3
+size 161533192

iter_0044/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

iter_0044/dual_variables.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d4834e9a2da1dc706ac5112e7fb16a34bd6821b3ed492f9a4262e7a14855713
+size 1020

iter_0044/log.json

@@ -0,0 +1,812 @@
+[
+  {
+    "iteration": 0,
+    "elapsed_s": 357.1625916957855,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7050087932900435,
+    "avg_questions": 0.75,
+    "avg_turns": 1.71875,
+    "ppo_loss": 0.007755006247732256,
+    "value_loss": 1.0904684109347207,
+    "approx_kl": 0.00942303503896775,
+    "clip_frac": 0.13966836780309677,
+    "entropy": 0.26119276721562656,
+    "kl_penalty": 0.012391226632254464,
+    "kl_per_seq": 0.012391226632254464,
+    "kl_seq_max": 0.16796875
+  },
+  {
+    "iteration": 1,
+    "elapsed_s": 343.5082986354828,
+    "lambda1": 0.009375000000000001,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.739140546953047,
+    "avg_questions": 0.84375,
+    "avg_turns": 1.84375,
+    "ppo_loss": -0.0010012539569288492,
+    "value_loss": 0.285459230421111,
+    "approx_kl": 0.006104676685936283,
+    "clip_frac": 0.08984375,
+    "entropy": 0.23781040566973388,
+    "kl_penalty": 0.011135101318359375,
+    "kl_per_seq": 0.011135101318359375,
+    "kl_seq_max": 0.136962890625
+  },
+  {
+    "iteration": 2,
+    "elapsed_s": 394.53891587257385,
+    "lambda1": 0.0125,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7890625,
+    "avg_questions": 0.78125,
+    "avg_turns": 1.78125,
+    "ppo_loss": 0.03891922673210502,
+    "value_loss": 0.637173940660432,
+    "approx_kl": 0.007546657929196954,
+    "clip_frac": 0.1015625,
+    "entropy": 0.24309116404037923,
+    "kl_penalty": 0.01812744140625,
+    "kl_per_seq": 0.01812744140625,
+    "kl_seq_max": 0.14404296875
+  },
+  {
+    "iteration": 3,
+    "elapsed_s": 300.50555062294006,
+    "lambda1": 0.0125,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7289949633699634,
+    "avg_questions": 0.75,
+    "avg_turns": 1.75,
+    "ppo_loss": -0.00402364440794502,
+    "value_loss": 0.3008941656776837,
+    "approx_kl": 0.0077281083379473,
+    "clip_frac": 0.12053571428571429,
+    "entropy": 0.23592129349708557,
+    "kl_penalty": 0.012259347098214286,
+    "kl_per_seq": 0.012259347098214286,
+    "kl_seq_max": 0.16559709821428573
+  },
+  {
+    "iteration": 4,
+    "elapsed_s": 367.04741644859314,
+    "lambda1": 0.00625,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.647762523934399,
+    "avg_questions": 0.6875,
+    "avg_turns": 1.6875,
+    "ppo_loss": 0.006391565016071711,
+    "value_loss": 0.4582739163722311,
+    "approx_kl": 0.010383476286993496,
+    "clip_frac": 0.13095238112977572,
+    "entropy": 0.2528929031853165,
+    "kl_penalty": 0.018035888671875,
+    "kl_per_seq": 0.018035888671875,
+    "kl_seq_max": 0.21428571428571427
+  },
+  {
+    "iteration": 5,
+    "elapsed_s": 382.89120960235596,
+    "lambda1": 0.00625,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.754916351010101,
+    "avg_questions": 0.75,
+    "avg_turns": 1.75,
+    "ppo_loss": 0.007902485584574086,
+    "value_loss": 0.31790804117918015,
+    "approx_kl": 0.005824520651783262,
+    "clip_frac": 0.07142857142857142,
+    "entropy": 0.25753957751606193,
+    "kl_penalty": 0.012311662946428572,
+    "kl_per_seq": 0.012311662946428572,
+    "kl_seq_max": 0.16071428571428573
+  },
+  {
+    "iteration": 6,
+    "elapsed_s": 291.7231960296631,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7139756944444444,
+    "avg_questions": 0.6875,
+    "avg_turns": 1.6875,
+    "ppo_loss": 0.01009868954618772,
+    "value_loss": 0.32571624716122943,
+    "approx_kl": 0.012746187547842661,
+    "clip_frac": 0.125,
+    "entropy": 0.24180377274751663,
+    "kl_penalty": 0.01416015625,
+    "kl_per_seq": 0.01416015625,
+    "kl_seq_max": 0.22916666666666666
+  },
+  {
+    "iteration": 7,
+    "elapsed_s": 340.50909781455994,
+    "lambda1": 0.00625,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.6727949134199136,
+    "avg_questions": 0.8125,
+    "avg_turns": 1.8125,
+    "ppo_loss": 0.007321253418922424,
+    "value_loss": 0.36242500762455165,
+    "approx_kl": 0.007387942634522915,
+    "clip_frac": 0.12890625,
+    "entropy": 0.27400176017545164,
+    "kl_penalty": 0.01043701171875,
+    "kl_per_seq": 0.01043701171875,
+    "kl_seq_max": 0.13134765625
+  },
+  {
+    "iteration": 8,
+    "elapsed_s": 330.3607187271118,
+    "lambda1": 0.0125,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.6581473214285714,
+    "avg_questions": 0.8125,
+    "avg_turns": 1.8125,
+    "ppo_loss": -0.0054003591649234295,
+    "value_loss": 0.37248116498813033,
+    "approx_kl": 0.007915348513051867,
+    "clip_frac": 0.13671875,
+    "entropy": 0.2505741387140006,
+    "kl_penalty": 0.0040740966796875,
+    "kl_per_seq": 0.0040740966796875,
+    "kl_seq_max": 0.1396484375
+  },
+  {
+    "iteration": 9,
+    "elapsed_s": 398.63353538513184,
+    "lambda1": 0.04375,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.6826368943556443,
+    "avg_questions": 1.0625,
+    "avg_turns": 1.90625,
+    "ppo_loss": -0.021302172332070768,
+    "value_loss": 1.4880596799775958,
+    "approx_kl": 0.010055532649857923,
+    "clip_frac": 0.15468750055879354,
+    "entropy": 0.2681027443613857,
+    "kl_penalty": 0.007171630859375,
+    "kl_per_seq": 0.007171630859375,
+    "kl_seq_max": 0.16552734375
+  },
+  {
+    "iteration": 10,
+    "elapsed_s": 333.02629804611206,
+    "lambda1": 0.049999999999999996,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.707235863095238,
+    "avg_questions": 0.8125,
+    "avg_turns": 1.78125,
+    "ppo_loss": -0.013005324872210622,
+    "value_loss": 0.39788789255544543,
+    "approx_kl": 0.006880114786326885,
+    "clip_frac": 0.12109375,
+    "entropy": 0.22961895586922765,
+    "kl_penalty": 3.0517578125e-05,
+    "kl_per_seq": 3.0517578125e-05,
+    "kl_seq_max": 0.128173828125
+  },
+  {
+    "iteration": 11,
+    "elapsed_s": 294.7535834312439,
+    "lambda1": 0.031249999999999993,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.8168402777777777,
+    "avg_questions": 0.5625,
+    "avg_turns": 1.5625,
+    "ppo_loss": -0.022718935273587704,
+    "value_loss": 0.3321453183889389,
+    "approx_kl": 0.006399596642170634,
+    "clip_frac": 0.10267857142857142,
+    "entropy": 0.18069506117275783,
+    "kl_penalty": 0.007045200892857143,
+    "kl_per_seq": 0.007045200892857143,
+    "kl_seq_max": 0.10574776785714286
+  },
+  {
+    "iteration": 12,
+    "elapsed_s": 343.0087254047394,
+    "lambda1": 0.03749999999999999,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.8053819444444443,
+    "avg_questions": 0.8125,
+    "avg_turns": 1.78125,
+    "ppo_loss": -0.020216253586113453,
+    "value_loss": 0.27846826042514294,
+    "approx_kl": 0.009821648010984063,
+    "clip_frac": 0.16796875,
+    "entropy": 0.25039526051841676,
+    "kl_penalty": 0.013916015625,
+    "kl_per_seq": 0.013916015625,
+    "kl_seq_max": 0.1962890625
+  },
+  {
+    "iteration": 13,
+    "elapsed_s": 363.77874422073364,
+    "lambda1": 0.05312499999999999,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.5087808892496393,
+    "avg_questions": 0.90625,
+    "avg_turns": 1.875,
+    "ppo_loss": -0.04679604054049209,
+    "value_loss": 0.3308154027219172,
+    "approx_kl": 0.013959913066139928,
+    "clip_frac": 0.16666666666666666,
+    "entropy": 0.2726931580238872,
+    "kl_penalty": 0.010181568287037037,
+    "kl_per_seq": 0.010181568287037037,
+    "kl_seq_max": 0.3732638888888889
+  },
+  {
+    "iteration": 14,
+    "elapsed_s": 352.3718099594116,
+    "lambda1": 0.05937499999999999,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.6545655733155732,
+    "avg_questions": 0.8125,
+    "avg_turns": 1.8125,
+    "ppo_loss": -0.03569141589105129,
+    "value_loss": 0.3559884512796998,
+    "approx_kl": 0.01543659484013915,
+    "clip_frac": 0.21875,
+    "entropy": 0.2455106396228075,
+    "kl_penalty": 0.01318359375,
+    "kl_per_seq": 0.01318359375,
+    "kl_seq_max": 0.3974609375
+  },
+  {
+    "iteration": 15,
+    "elapsed_s": 313.32435488700867,
+    "lambda1": 0.05312499999999999,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7722470238095239,
+    "avg_questions": 0.6875,
+    "avg_turns": 1.65625,
+    "ppo_loss": -0.0331129701091693,
+    "value_loss": 0.407362831899753,
+    "approx_kl": 0.014424336071197804,
+    "clip_frac": 0.18173076957464218,
+    "entropy": 0.23549939606052178,
+    "kl_penalty": 0.022601787860576924,
+    "kl_per_seq": 0.022601787860576924,
+    "kl_seq_max": 0.5504807692307693
+  },
+  {
+    "iteration": 16,
+    "elapsed_s": 276.3943181037903,
+    "lambda1": 0.06562499999999999,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.8410790598290598,
+    "avg_questions": 0.875,
+    "avg_turns": 1.84375,
+    "ppo_loss": -0.037679088301956654,
+    "value_loss": 0.20089253911282867,
+    "approx_kl": 0.011464653187431395,
+    "clip_frac": 0.1510416679084301,
+    "entropy": 0.20766122685745358,
+    "kl_penalty": 0.06768798828125,
+    "kl_per_seq": 0.06768798828125,
+    "kl_seq_max": 0.5673828125
+  },
+  {
+    "iteration": 17,
+    "elapsed_s": 375.74297308921814,
+    "lambda1": 0.0875,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.8539749313186813,
+    "avg_questions": 0.96875,
+    "avg_turns": 1.90625,
+    "ppo_loss": -0.0474210566530625,
+    "value_loss": 0.2925632045293848,
+    "approx_kl": 0.017900955832252898,
+    "clip_frac": 0.28083333373069763,
+    "entropy": 0.23634420384963353,
+    "kl_penalty": 0.009114583333333334,
+    "kl_per_seq": 0.009114583333333334,
+    "kl_seq_max": 0.5213541666666667
+  },
+  {
+    "iteration": 18,
+    "elapsed_s": 330.67957067489624,
+    "lambda1": 0.10625,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7826446123321124,
+    "avg_questions": 0.9375,
+    "avg_turns": 1.90625,
+    "ppo_loss": -0.055612206557079366,
+    "value_loss": 0.31383695316157845,
+    "approx_kl": 0.016603972174619373,
+    "clip_frac": 0.24210526362845772,
+    "entropy": 0.2367225446199116,
+    "kl_penalty": 0.05126953125,
+    "kl_per_seq": 0.05126953125,
+    "kl_seq_max": 0.6118421052631579
+  },
+  {
+    "iteration": 19,
+    "elapsed_s": 316.73736214637756,
+    "lambda1": 0.10625,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7678131764069264,
+    "avg_questions": 0.75,
+    "avg_turns": 1.71875,
+    "ppo_loss": -0.05081604502123335,
+    "value_loss": 0.3012801162574602,
+    "approx_kl": 0.015763986157253385,
+    "clip_frac": 0.16459627384724823,
+    "entropy": 0.21080296000708704,
+    "kl_penalty": 0.10081946331521739,
+    "kl_per_seq": 0.10081946331521739,
+    "kl_seq_max": 0.7853260869565217
+  },
+  {
+    "iteration": 20,
+    "elapsed_s": 292.9455828666687,
+    "lambda1": 0.1125,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7923935439560439,
+    "avg_questions": 0.8125,
+    "avg_turns": 1.75,
+    "ppo_loss": -0.03436802277962367,
+    "value_loss": 0.327542544901371,
+    "approx_kl": 0.016658279796441397,
+    "clip_frac": 0.2,
+    "entropy": 0.2346005121866862,
+    "kl_penalty": 0.07431640625,
+    "kl_per_seq": 0.07431640625,
+    "kl_seq_max": 0.7651041666666667
+  },
+  {
+    "iteration": 21,
+    "elapsed_s": 316.87915897369385,
+    "lambda1": 0.10625,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7417455808080807,
+    "avg_questions": 0.6875,
+    "avg_turns": 1.6875,
+    "ppo_loss": -0.045961443748739034,
+    "value_loss": 0.4759538339243995,
+    "approx_kl": 0.015832876993550196,
+    "clip_frac": 0.22685185240374672,
+    "entropy": 0.2288952949974272,
+    "kl_penalty": 0.08394368489583333,
+    "kl_per_seq": 0.08394368489583333,
+    "kl_seq_max": 0.7795138888888888
+  },
+  {
+    "iteration": 22,
+    "elapsed_s": 305.55787348747253,
+    "lambda1": 0.09999999999999999,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.8609916125541125,
+    "avg_questions": 0.6875,
+    "avg_turns": 1.6875,
+    "ppo_loss": -0.05956608028366016,
+    "value_loss": 0.24114569362539512,
+    "approx_kl": 0.019050200308601443,
+    "clip_frac": 0.24839743685263854,
+    "entropy": 0.21138634217473176,
+    "kl_penalty": 0.06511981670673077,
+    "kl_per_seq": 0.06511981670673077,
+    "kl_seq_max": 0.8031850961538461
+  },
+  {
+    "iteration": 23,
+    "elapsed_s": 321.8149151802063,
+    "lambda1": 0.109375,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7557539682539682,
+    "avg_questions": 0.84375,
+    "avg_turns": 1.8125,
+    "ppo_loss": -0.056632211245596406,
+    "value_loss": 0.3020759107545018,
+    "approx_kl": 0.01828147955238819,
+    "clip_frac": 0.3,
+    "entropy": 0.24711562618613242,
+    "kl_penalty": 0.101806640625,
+    "kl_per_seq": 0.101806640625,
+    "kl_seq_max": 0.84375
+  },
+  {
+    "iteration": 24,
+    "elapsed_s": 306.5719954967499,
+    "lambda1": 0.109375,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7853937728937729,
+    "avg_questions": 0.75,
+    "avg_turns": 1.75,
+    "ppo_loss": -0.037689958016077676,
+    "value_loss": 0.39611613551775615,
+    "approx_kl": 0.01130907932917277,
+    "clip_frac": 0.15,
+    "entropy": 0.1994072715441386,
+    "kl_penalty": 0.18115234375,
+    "kl_per_seq": 0.18115234375,
+    "kl_seq_max": 1.1114583333333334
+  },
+  {
+    "iteration": 25,
+    "elapsed_s": 328.52625131607056,
+    "lambda1": 0.103125,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.6865439248251748,
+    "avg_questions": 0.6875,
+    "avg_turns": 1.65625,
+    "ppo_loss": -0.04527500409044718,
+    "value_loss": 0.47288530751278524,
+    "approx_kl": 0.015802607822575067,
+    "clip_frac": 0.19868421084002444,
+    "entropy": 0.20268321821564123,
+    "kl_penalty": 0.16164679276315788,
+    "kl_per_seq": 0.16164679276315788,
+    "kl_seq_max": 1.0608552631578947
+  },
+  {
+    "iteration": 26,
+    "elapsed_s": 301.34392499923706,
+    "lambda1": 0.10625,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7395833333333334,
+    "avg_questions": 0.78125,
+    "avg_turns": 1.6875,
+    "ppo_loss": -0.04237345000728965,
+    "value_loss": 0.4161538938060403,
+    "approx_kl": 0.015270014468114823,
+    "clip_frac": 0.15625000093132257,
+    "entropy": 0.1977705955505371,
+    "kl_penalty": 0.2110595703125,
+    "kl_per_seq": 0.2110595703125,
+    "kl_seq_max": 1.24658203125
+  },
+  {
+    "iteration": 27,
+    "elapsed_s": 348.94201493263245,
+    "lambda1": 0.1125,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7194080225330226,
+    "avg_questions": 0.8125,
+    "avg_turns": 1.8125,
+    "ppo_loss": -0.05180658517699493,
+    "value_loss": 0.2778499259760505,
+    "approx_kl": 0.01625884206671464,
+    "clip_frac": 0.19736842105263158,
+    "entropy": 0.21087203449324557,
+    "kl_penalty": 0.13524105674342105,
+    "kl_per_seq": 0.13524105674342105,
+    "kl_seq_max": 1.1488486842105263
+  },
+  {
+    "iteration": 28,
+    "elapsed_s": 322.038512468338,
+    "lambda1": 0.109375,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.6542670003607503,
+    "avg_questions": 0.71875,
+    "avg_turns": 1.65625,
+    "ppo_loss": -0.04524578066433177,
+    "value_loss": 0.4445836964775534,
+    "approx_kl": 0.019655843183179113,
+    "clip_frac": 0.21176470640827627,
+    "entropy": 0.21890325931941762,
+    "kl_penalty": 0.08786190257352941,
+    "kl_per_seq": 0.08786190257352941,
+    "kl_seq_max": 0.859375
+  },
+  {
+    "iteration": 29,
+    "elapsed_s": 339.1300230026245,
+    "lambda1": 0.096875,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7828609758297258,
+    "avg_questions": 0.625,
+    "avg_turns": 1.625,
+    "ppo_loss": -0.059186975226590506,
+    "value_loss": 0.31758245355204534,
+    "approx_kl": 0.018988314819963353,
+    "clip_frac": 0.23684210526315788,
+    "entropy": 0.20613071636149757,
+    "kl_penalty": 0.34781044407894735,
+    "kl_per_seq": 0.34781044407894735,
+    "kl_seq_max": 1.433388157894737
+  },
+  {
+    "iteration": 30,
+    "elapsed_s": 290.3177366256714,
+    "lambda1": 0.084375,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.684590583028083,
+    "avg_questions": 0.625,
+    "avg_turns": 1.625,
+    "ppo_loss": -0.039527233276102275,
+    "value_loss": 0.35584185810552704,
+    "approx_kl": 0.021562429351939097,
+    "clip_frac": 0.2777777777777778,
+    "entropy": 0.23942414422829947,
+    "kl_penalty": 0.15489366319444445,
+    "kl_per_seq": 0.15489366319444445,
+    "kl_seq_max": 1.1883680555555556
+  },
+  {
+    "iteration": 31,
+    "elapsed_s": 299.3714759349823,
+    "lambda1": 0.084375,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.875,
+    "avg_questions": 0.75,
+    "avg_turns": 1.6875,
+    "ppo_loss": -0.044965135399252176,
+    "value_loss": 0.2800211445428431,
+    "approx_kl": 0.018849731306545436,
+    "clip_frac": 0.1875,
+    "entropy": 0.2140436777845025,
+    "kl_penalty": 0.294921875,
+    "kl_per_seq": 0.294921875,
+    "kl_seq_max": 1.630859375
+  },
+  {
+    "iteration": 32,
+    "elapsed_s": 351.7063624858856,
+    "lambda1": 0.078125,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7589318667443667,
+    "avg_questions": 0.6875,
+    "avg_turns": 1.6875,
+    "ppo_loss": -0.03741050139069557,
+    "value_loss": 0.28676399641803335,
+    "approx_kl": 0.023076566202299937,
+    "clip_frac": 0.24702381023338862,
+    "entropy": 0.21046061547739164,
+    "kl_penalty": 0.3627232142857143,
+    "kl_per_seq": 0.3627232142857143,
+    "kl_seq_max": 1.9888392857142858
+  },
+  {
+    "iteration": 33,
+    "elapsed_s": 402.1016628742218,
+    "lambda1": 0.075,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.827605294011544,
+    "avg_questions": 0.71875,
+    "avg_turns": 1.71875,
+    "ppo_loss": -0.040155550331941674,
+    "value_loss": 0.20966205213751113,
+    "approx_kl": 0.017562197853944132,
+    "clip_frac": 0.21811224626643316,
+    "entropy": 0.21647065665040696,
+    "kl_penalty": 0.28662109375,
+    "kl_per_seq": 0.28662109375,
+    "kl_seq_max": 2.1021205357142856
+  },
+  {
+    "iteration": 34,
+    "elapsed_s": 324.4843943119049,
+    "lambda1": 0.08125,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7170138888888888,
+    "avg_questions": 0.8125,
+    "avg_turns": 1.75,
+    "ppo_loss": -0.030176887458020992,
+    "value_loss": 0.3569460050626235,
+    "approx_kl": 0.019110250879417767,
+    "clip_frac": 0.22727272727272727,
+    "entropy": 0.20825903794982217,
+    "kl_penalty": 0.5365767045454546,
+    "kl_per_seq": 0.5365767045454546,
+    "kl_seq_max": 2.335227272727273
+  },
+  {
+    "iteration": 35,
+    "elapsed_s": 315.85867977142334,
+    "lambda1": 0.06875,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7331473214285715,
+    "avg_questions": 0.625,
+    "avg_turns": 1.53125,
+    "ppo_loss": -0.006593746634630056,
+    "value_loss": 0.4972928464412689,
+    "approx_kl": 0.015867754243887387,
+    "clip_frac": 0.19230769230769232,
+    "entropy": 0.23934545998389906,
+    "kl_penalty": 0.43111478365384615,
+    "kl_per_seq": 0.43111478365384615,
+    "kl_seq_max": 2.6298076923076925
+  },
+  {
+    "iteration": 36,
+    "elapsed_s": 386.9313094615936,
+    "lambda1": 0.084375,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.8004261363636361,
+    "avg_questions": 0.90625,
+    "avg_turns": 1.84375,
+    "ppo_loss": 0.003060997887091203,
+    "value_loss": 0.30295307460156357,
+    "approx_kl": 0.01550799777561968,
+    "clip_frac": 0.2121212130243128,
+    "entropy": 0.22590754248879172,
+    "kl_penalty": 0.33540482954545453,
+    "kl_per_seq": 0.33540482954545453,
+    "kl_seq_max": 2.1988636363636362
+  },
+  {
+    "iteration": 37,
+    "elapsed_s": 372.7559549808502,
+    "lambda1": 0.078125,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7559456168831168,
+    "avg_questions": 0.6875,
+    "avg_turns": 1.65625,
+    "ppo_loss": -0.041432250601549946,
+    "value_loss": 0.3984727369000514,
+    "approx_kl": 0.024890107413132984,
+    "clip_frac": 0.2562500002483527,
+    "entropy": 0.22201429742077985,
+    "kl_penalty": 0.7021484375,
+    "kl_per_seq": 0.7021484375,
+    "kl_seq_max": 3.1744791666666665
+  },
+  {
+    "iteration": 38,
+    "elapsed_s": 349.12115454673767,
+    "lambda1": 0.06875,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7446377840909091,
+    "avg_questions": 0.65625,
+    "avg_turns": 1.65625,
+    "ppo_loss": -0.011596236005425453,
+    "value_loss": 0.3541109532117844,
+    "approx_kl": 0.01865895972587168,
+    "clip_frac": 0.175,
+    "entropy": 0.21150051951408386,
+    "kl_penalty": 0.7384765625,
+    "kl_per_seq": 0.7384765625,
+    "kl_seq_max": 3.715625
+  },
+  {
+    "iteration": 39,
+    "elapsed_s": 405.69459891319275,
+    "lambda1": 0.05,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7032117361804863,
+    "avg_questions": 0.5625,
+    "avg_turns": 1.5625,
+    "ppo_loss": 0.028779844088213786,
+    "value_loss": 0.3979421172823225,
+    "approx_kl": 0.018495287214006697,
+    "clip_frac": 0.21428571428571427,
+    "entropy": 0.18224240839481354,
+    "kl_penalty": 1.3233816964285714,
+    "kl_per_seq": 1.3233816964285714,
+    "kl_seq_max": 3.21875
+  },
+  {
+    "iteration": 40,
+    "elapsed_s": 477.3596832752228,
+    "lambda1": 0.037500000000000006,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7178388451825952,
+    "avg_questions": 0.625,
+    "avg_turns": 1.5625,
+    "ppo_loss": -0.053192346141888544,
+    "value_loss": 0.4054635831942925,
+    "approx_kl": 0.019211487128184393,
+    "clip_frac": 0.28846153846153844,
+    "entropy": 0.1979089849270307,
+    "kl_penalty": 1.4969951923076923,
+    "kl_per_seq": 1.4969951923076923,
+    "kl_seq_max": 4.668269230769231
+  },
+  {
+    "iteration": 41,
+    "elapsed_s": 529.4953544139862,
+    "lambda1": 0.0062500000000000056,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.8039234203296702,
+    "avg_questions": 0.4375,
+    "avg_turns": 1.4375,
+    "ppo_loss": -0.010720850278933844,
+    "value_loss": 0.440558356543382,
+    "approx_kl": 0.015522866044193506,
+    "clip_frac": 0.20138889054457346,
+    "entropy": 0.15809335187077522,
+    "kl_penalty": 1.9837239583333333,
+    "kl_per_seq": 1.9837239583333333,
+    "kl_seq_max": 5.434895833333333
+  },
+  {
+    "iteration": 42,
+    "elapsed_s": 399.98438239097595,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7863238324175824,
+    "avg_questions": 0.59375,
+    "avg_turns": 1.5625,
+    "ppo_loss": -0.019172271092732746,
+    "value_loss": 0.38196369177765316,
+    "approx_kl": 0.015519324276182387,
+    "clip_frac": 0.2638888888888889,
+    "entropy": 0.20362108945846558,
+    "kl_penalty": 1.5021701388888888,
+    "kl_per_seq": 1.5021701388888888,
+    "kl_seq_max": 3.8177083333333335
+  },
+  {
+    "iteration": 43,
+    "elapsed_s": 382.99004435539246,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7528521825396826,
+    "avg_questions": 0.625,
+    "avg_turns": 1.59375,
+    "ppo_loss": -0.04032935032790357,
+    "value_loss": 0.3547650304707614,
+    "approx_kl": 0.016969902102242817,
+    "clip_frac": 0.18939394029704007,
+    "entropy": 0.20878374915231357,
+    "kl_penalty": 0.7615411931818182,
+    "kl_per_seq": 0.7615411931818182,
+    "kl_seq_max": 3.0369318181818183
+  },
+  {
+    "iteration": 44,
+    "elapsed_s": 357.9368350505829,
+    "lambda1": 0.0,
+    "lambda2": 0.0,
+    "n_episodes": 32,
+    "avg_reward": 0.7684809981684981,
+    "avg_questions": 0.6875,
+    "avg_turns": 1.53125,
+    "ppo_loss": -0.018041244623335926,
+    "value_loss": 0.4228832056661221,
+    "approx_kl": 0.014606282114982605,
+    "clip_frac": 0.125,
+    "entropy": 0.17023578387769786,
+    "kl_penalty": 1.1877663352272727,
+    "kl_per_seq": 1.1877663352272727,
+    "kl_seq_max": 4.089488636363637
+  }
+]

iter_0044/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dbb9ccc29b4517e9dccf26d1b9f1bb48113a6eb969a61c50ba9c8a2b0089d586
+size 11421990

iter_0044/tokenizer_config.json

@@ -0,0 +1,31 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "left",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "truncation_side": "left",
+  "unk_token": null
+}

iter_0044/train_state.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:404594e0d10049658ec4ed7766d784a2df64113a6af1ca84cff56568cec9f141
+size 104946244

iter_0044/value_heads.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4ba54ab385f1ddd54cb22c7b22c2616c8ede66017918f2746c8792b0cf9035a
+size 44096056