DPO emergency upload (timeout) - job 4c139833-f70c-49d9-b684-84c0688eb857

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,209 @@
+---
+base_model: unsloth/gpt-oss-20b
+library_name: peft
+tags:
+- base_model:adapter:unsloth/gpt-oss-20b
+- dpo
+- lora
+- transformers
+- trl
+- unsloth
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.17.1

adapter_config.json

@@ -0,0 +1,45 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": {
+    "base_model_class": "GptOssForCausalLM",
+    "parent_library": "transformers.models.gpt_oss.modeling_gpt_oss"
+  },
+  "base_model_name_or_path": "unsloth/gpt-oss-20b",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "k_proj",
+    "up_proj",
+    "o_proj",
+    "gate_proj",
+    "q_proj",
+    "down_proj"
+  ],
+  "target_parameters": null,
+  "task_type": null,
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97dca5618b69b45c0ff2257182550e3efd5a5e987e3bfb7bdf938aabdd070fae
+size 63726760

chat_template.jinja

@@ -0,0 +1,345 @@
+{# Chat template fixes by Unsloth #}
+{#-
+  In addition to the normal inputs of `messages` and `tools`, this template also accepts the
+  following kwargs:
+  - "builtin_tools": A list, can contain "browser" and/or "python".
+  - "model_identity": A string that optionally describes the model identity.
+  - "reasoning_effort": A string that describes the reasoning effort, defaults to "medium".
+ #}
+
+{#- Tool Definition Rendering ============================================== #}
+{%- macro render_typescript_type(param_spec, required_params, is_nullable=false) -%}
+    {%- if param_spec.type == "array" -%}
+        {%- if param_spec['items'] -%}
+            {%- if param_spec['items']['type'] == "string" -%}
+                {{- "string[]" }}
+            {%- elif param_spec['items']['type'] == "number" -%}
+                {{- "number[]" }}
+            {%- elif param_spec['items']['type'] == "integer" -%}
+                {{- "number[]" }}
+            {%- elif param_spec['items']['type'] == "boolean" -%}
+                {{- "boolean[]" }}
+            {%- else -%}
+                {%- set inner_type = render_typescript_type(param_spec['items'], required_params) -%}
+                {%- if inner_type == "object | object" or inner_type|length > 50 -%}
+                    {{- "any[]" }}
+                {%- else -%}
+                    {{- inner_type + "[]" }}
+                {%- endif -%}
+            {%- endif -%}
+            {%- if param_spec.nullable -%}
+                {{- " | null" }}
+            {%- endif -%}
+        {%- else -%}
+            {{- "any[]" }}
+            {%- if param_spec.nullable -%}
+                {{- " | null" }}
+            {%- endif -%}
+        {%- endif -%}
+    {%- elif param_spec.type is defined and param_spec.type is iterable and param_spec.type is not string and param_spec.type is not mapping and param_spec.type[0] is defined -%}
+        {#- Handle array of types like ["object", "object"] from Union[dict, list] #}
+        {%- if param_spec.type | length > 1 -%}
+            {{- param_spec.type | join(" | ") }}
+        {%- else -%}
+            {{- param_spec.type[0] }}
+        {%- endif -%}
+    {%- elif param_spec.oneOf -%}
+        {#- Handle oneOf schemas - check for complex unions and fallback to any #}
+        {%- set has_object_variants = false -%}
+        {%- for variant in param_spec.oneOf -%}
+            {%- if variant.type == "object" -%}
+                {%- set has_object_variants = true -%}
+            {%- endif -%}
+        {%- endfor -%}
+        {%- if has_object_variants and param_spec.oneOf|length > 1 -%}
+            {{- "any" }}
+        {%- else -%}
+            {%- for variant in param_spec.oneOf -%}
+                {{- render_typescript_type(variant, required_params) -}}
+                {%- if variant.description %}
+                    {{- "// " + variant.description }}
+                {%- endif -%}
+                {%- if variant.default is defined %}
+                    {{ "// default: " + variant.default|tojson }}
+                {%- endif -%}
+                {%- if not loop.last %}
+                    {{- " | " }}
+                {% endif -%}
+            {%- endfor -%}
+        {%- endif -%}
+    {%- elif param_spec.type == "string" -%}
+        {%- if param_spec.enum -%}
+            {{- '"' + param_spec.enum|join('" | "') + '"' -}}
+        {%- else -%}
+            {{- "string" }}
+            {%- if param_spec.nullable %}
+                {{- " | null" }}
+            {%- endif -%}
+        {%- endif -%}
+    {%- elif param_spec.type == "number" -%}
+        {{- "number" }}
+    {%- elif param_spec.type == "integer" -%}
+        {{- "number" }}
+    {%- elif param_spec.type == "boolean" -%}
+        {{- "boolean" }}
+
+    {%- elif param_spec.type == "object" -%}
+        {%- if param_spec.properties -%}
+            {{- "{\n" }}
+            {%- for prop_name, prop_spec in param_spec.properties.items() -%}
+                {{- prop_name -}}
+                {%- if prop_name not in (param_spec.required or []) -%}
+                    {{- "?" }}
+                {%- endif -%}
+                {{- ": " }}
+                {{ render_typescript_type(prop_spec, param_spec.required or []) }}
+                {%- if not loop.last -%}
+                    {{-", " }}
+                {%- endif -%}
+            {%- endfor -%}
+            {{- "}" }}
+        {%- else -%}
+            {{- "object" }}
+        {%- endif -%}
+    {%- else -%}
+        {{- "any" }}
+    {%- endif -%}
+{%- endmacro -%}
+
+{%- macro render_tool_namespace(namespace_name, tools) -%}
+    {{- "## " + namespace_name + "\n\n" }}
+    {{- "namespace " + namespace_name + " {\n\n" }}
+    {%- for tool in tools %}
+        {%- set tool = tool.function %}
+        {{- "// " + tool.description + "\n" }}
+        {{- "type "+ tool.name + " = " }}
+        {%- if tool.parameters and tool.parameters.properties %}
+            {{- "(_: {\n" }}
+            {%- for param_name, param_spec in tool.parameters.properties.items() %}
+                {%- if param_spec.description %}
+                    {{- "// " + param_spec.description + "\n" }}
+                {%- endif %}
+                {{- param_name }}
+                {%- if param_name not in (tool.parameters.required or []) -%}
+                    {{- "?" }}
+                {%- endif -%}
+                {{- ": " }}
+                {{- render_typescript_type(param_spec, tool.parameters.required or []) }}
+                {%- if param_spec.default is defined -%}
+                    {%- if param_spec.enum %}
+                        {{- ", // default: " + param_spec.default }}
+                    {%- elif param_spec.oneOf %}
+                        {{- "// default: " + param_spec.default }}
+                    {%- else %}
+                        {{- ", // default: " + param_spec.default|tojson }}
+                    {%- endif -%}
+                {%- endif -%}
+                {%- if not loop.last %}
+                    {{- ",\n" }}
+                {%- else %}
+                    {{- ",\n" }}
+                {%- endif -%}
+            {%- endfor %}
+            {{- "}) => any;\n\n" }}
+        {%- else -%}
+            {{- "() => any;\n\n" }}
+        {%- endif -%}
+    {%- endfor %}
+    {{- "} // namespace " + namespace_name }}
+{%- endmacro -%}
+
+{%- macro render_builtin_tools(browser_tool, python_tool) -%}
+    {%- if browser_tool %}
+        {{- "## browser\n\n" }}
+        {{- "// Tool for browsing.\n" }}
+        {{- "// The `cursor` appears in brackets before each browsing display: `[{cursor}]`.\n" }}
+        {{- "// Cite information from the tool using the following format:\n" }}
+        {{- "// `【{cursor}†L{line_start}(-L{line_end})?】`, for example: `【6†L9-L11】` or `【8†L3】`.\n" }}
+        {{- "// Do not quote more than 10 words directly from the tool output.\n" }}
+        {{- "// sources=web (default: web)\n" }}
+        {{- "namespace browser {\n\n" }}
+        {{- "// Searches for information related to `query` and displays `topn` results.\n" }}
+        {{- "type search = (_: {\n" }}
+        {{- "query: string,\n" }}
+        {{- "topn?: number, // default: 10\n" }}
+        {{- "source?: string,\n" }}
+        {{- "}) => any;\n\n" }}
+        {{- "// Opens the link `id` from the page indicated by `cursor` starting at line number `loc`, showing `num_lines` lines.\n" }}
+        {{- "// Valid link ids are displayed with the formatting: `【{id}†.*】`.\n" }}
+        {{- "// If `cursor` is not provided, the most recent page is implied.\n" }}
+        {{- "// If `id` is a string, it is treated as a fully qualified URL associated with `source`.\n" }}
+        {{- "// If `loc` is not provided, the viewport will be positioned at the beginning of the document or centered on the most relevant passage, if available.\n" }}
+        {{- "// Use this function without `id` to scroll to a new location of an opened page.\n" }}
+        {{- "type open = (_: {\n" }}
+        {{- "id?: number | string, // default: -1\n" }}
+        {{- "cursor?: number, // default: -1\n" }}
+        {{- "loc?: number, // default: -1\n" }}
+        {{- "num_lines?: number, // default: -1\n" }}
+        {{- "view_source?: boolean, // default: false\n" }}
+        {{- "source?: string,\n" }}
+        {{- "}) => any;\n\n" }}
+        {{- "// Finds exact matches of `pattern` in the current page, or the page given by `cursor`.\n" }}
+        {{- "type find = (_: {\n" }}
+        {{- "pattern: string,\n" }}
+        {{- "cursor?: number, // default: -1\n" }}
+        {{- "}) => any;\n\n" }}
+        {{- "} // namespace browser\n\n" }}
+    {%- endif -%}
+
+    {%- if python_tool %}
+        {{- "## python\n\n" }}
+        {{- "Use this tool to execute Python code in your chain of thought. The code will not be shown to the user. This tool should be used for internal reasoning, but not for code that is intended to be visible to the user (e.g. when creating plots, tables, or files).\n\n" }}
+        {{- "When you send a message containing Python code to python, it will be executed in a stateful Jupyter notebook environment. python will respond with the output of the execution or time out after 120.0 seconds. The drive at '/mnt/data' can be used to save and persist user files. Internet access for this session is UNKNOWN. Depends on the cluster.\n\n" }}
+    {%- endif -%}
+{%- endmacro -%}
+
+{#- System Message Construction ============================================ #}
+{%- macro build_system_message() -%}
+    {%- if model_identity is not defined %}
+        {%- set model_identity = "You are ChatGPT, a large language model trained by OpenAI." %}
+    {%- endif %}
+    {{- model_identity + "\n" }}
+    {{- "Knowledge cutoff: 2024-06\n" }}
+    {{- "Current date: " + strftime_now("%Y-%m-%d") + "\n\n" }}
+    {%- if reasoning_effort is not defined %}
+        {%- set reasoning_effort = "medium" %}
+    {%- endif %}
+    {{- "Reasoning: " + reasoning_effort + "\n\n" }}
+    {%- if builtin_tools is defined and builtin_tools is not none %}
+        {{- "# Tools\n\n" }}
+        {%- set available_builtin_tools = namespace(browser=false, python=false) %}
+        {%- for tool in builtin_tools %}
+            {%- if tool == "browser" %}
+                {%- set available_builtin_tools.browser = true %}
+            {%- elif tool == "python" %}
+                {%- set available_builtin_tools.python = true %}
+            {%- endif %}
+        {%- endfor %}
+        {{- render_builtin_tools(available_builtin_tools.browser, available_builtin_tools.python) }}
+    {%- endif -%}
+    {{- "# Valid channels: analysis, commentary, final. Channel must be included for every message." }}
+    {%- if tools -%}
+        {{- "\nCalls to these tools must go to the commentary channel: 'functions'." }}
+    {%- endif -%}
+{%- endmacro -%}
+
+{#- Main Template Logic ================================================= #}
+{#- Set defaults #}
+
+{#- Render system message #}
+{{- "<|start|>system<|message|>" }}
+{{- build_system_message() }}
+{{- "<|end|>" }}
+
+{#- Extract developer message #}
+{%- if developer_instructions is defined and developer_instructions is not none %}
+    {%- set developer_message = developer_instructions %}
+    {%- set loop_messages = messages %}
+{%- elif messages[0].role == "developer" or messages[0].role == "system" %}
+    {%- set developer_message = messages[0].content %}
+    {%- set loop_messages = messages[1:] %}
+{%- else %}
+    {%- set developer_message = "" %}
+    {%- set loop_messages = messages %}
+{%- endif %}
+
+{#- Render developer message #}
+{%- if developer_message or tools %}
+    {{- "<|start|>developer<|message|>" }}
+    {%- if developer_message %}
+        {{- "# Instructions\n\n" }}
+        {{- developer_message }}
+    {%- endif %}
+    {%- if tools -%}
+        {%- if developer_message %}
+            {{- "\n\n" }}
+        {%- endif %}
+        {{- "# Tools\n\n" }}
+        {{- render_tool_namespace("functions", tools) }}
+    {%- endif -%}
+    {{- "<|end|>" }}
+{%- endif %}
+
+{#- Render messages #}
+{%- set last_tool_call = namespace(name=none) %}
+{%- for message in loop_messages -%}
+    {#- At this point only assistant/user/tool messages should remain #}
+    {%- if message.role == 'assistant' -%}
+        {#- Checks to ensure the messages are being passed in the format we expect #}
+        {%- if "thinking" in message %}
+            {%- if "<|channel|>analysis<|message|>" in message.thinking or "<|channel|>final<|message|>" in message.thinking %}
+                {{- raise_exception("You have passed a message containing <|channel|> tags in the thinking field. Instead of doing this, you should pass analysis messages (the string between '<|message|>' and '<|end|>') in the 'thinking' field, and final messages (the string between '<|message|>' and '<|end|>') in the 'content' field.") }}
+            {%- endif %}
+        {%- endif %}
+        {%- if "tool_calls" in message %}
+            {#- We need very careful handling here - we want to drop the tool call analysis message if the model #}
+            {#- has output a later <|final|> message, but otherwise we want to retain it. This is the only case #}
+            {#- when we render CoT/analysis messages in inference. #}
+            {%- set future_final_message = namespace(found=false) %}
+            {%- for future_message in loop_messages[loop.index:] %}
+                {%- if future_message.role == 'assistant' and "tool_calls" not in future_message %}
+                    {%- set future_final_message.found = true %}
+                {%- endif %}
+            {%- endfor %}
+            {#- We assume max 1 tool call per message, and so we infer the tool call name #}
+            {#- in "tool" messages from the most recent assistant tool call name #}
+            {%- set tool_call = message.tool_calls[0] %}
+            {%- if tool_call.function %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {%- if message.content and message.thinking %}
+                {{- raise_exception("Cannot pass both content and thinking in an assistant message with tool calls! Put the analysis message in one or the other, but not both.") }}
+            {%- elif message.content and not future_final_message.found %}
+                {{- "<|start|>assistant<|channel|>analysis<|message|>" + message.content + "<|end|>" }}
+            {%- elif message.thinking and not future_final_message.found %}
+                {{- "<|start|>assistant<|channel|>analysis<|message|>" + message.thinking + "<|end|>" }}
+            {%- endif %}
+            {{- "<|start|>assistant to=" }}
+            {{- "functions." + tool_call.name + "<|channel|>commentary " }}
+            {{- (tool_call.content_type if tool_call.content_type is defined else "json") + "<|message|>" }}
+            {%- if tool_call.arguments is string %}
+                {{- tool_call.arguments }}
+            {%- else %}
+                {{- tool_call.arguments|tojson }}
+            {%- endif %}
+            {{- "<|call|>" }}
+            {%- set last_tool_call.name = tool_call.name %}
+        {%- elif loop.last and not add_generation_prompt %}
+            {#- Only render the CoT if the final turn is an assistant turn and add_generation_prompt is false #}
+            {#- This is a situation that should only occur in training, never in inference. #}
+            {%- if "thinking" in message %}
+                {{- "<|start|>assistant<|channel|>analysis<|message|>" + message.thinking + "<|end|>" }}
+            {%- endif %}
+            {#- <|return|> indicates the end of generation, but <|end|> does not #}
+            {#- <|return|> should never be an input to the model, but we include it as the final token #}
+            {#- when training, so the model learns to emit it. #}
+            {{- "<|start|>assistant<|channel|>final<|message|>" + message.content + "<|end|>" }}
+        {%- elif "thinking" in message %}
+            {#- CoT is dropped during all previous turns, so we never render it for inference #}
+            {{- "<|start|>assistant<|channel|>analysis<|message|>" + message.thinking + "<|end|>" }}
+            {%- set last_tool_call.name = none %}
+        {%- else %}
+            {#- CoT is dropped during all previous turns, so we never render it for inference #}
+            {{- "<|start|>assistant<|channel|>final<|message|>" + message.content + "<|end|>" }}
+            {%- set last_tool_call.name = none %}
+        {%- endif %}
+    {%- elif message.role == 'tool' -%}
+        {%- if last_tool_call.name is none %}
+            {{- raise_exception("Message has tool role, but there was no previous assistant message with a tool call!") }}
+        {%- endif %}
+        {{- "<|start|>functions." + last_tool_call.name }}
+        {%- if message.content is string %}
+            {{- " to=assistant<|channel|>commentary<|message|>" + message.content + "<|end|>" }}
+        {%- else %}
+            {{- " to=assistant<|channel|>commentary<|message|>" + message.content|tojson + "<|end|>" }}
+        {%- endif %}
+    {%- elif message.role == 'user' -%}
+        {{- "<|start|>user<|message|>" + message.content + "<|end|>" }}
+    {%- endif -%}
+{%- endfor -%}
+
+{#- Generation prompt #}
+{%- if add_generation_prompt -%}
+<|start|>assistant
+{%- endif -%}
+{# Copyright 2025-present Unsloth. Apache 2.0 License. Unsloth chat template fixes. Edited from ggml-org & OpenAI #}

loss.txt

@@ -0,0 +1 @@
+156,no_eval

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40d6cf5e0df049b7eaa805bff8ed8cc7205d8f4d23ba482f8df8fdd52c576c04
+size 32569658

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ba77c4358d5913436545fc6995706402cd54ccf015646708e622eca7f93ed87
+size 14244

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09d174ff6de6f33125a8a81084fa043de37ad6b4d22ff40a8983c3c934efaf0c
+size 1064

special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<|startoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|return|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|reserved_200017|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0614fe83cadab421296e664e1f48f4261fa8fef6e03e63bb75c20f38e37d07d3
+size 27868174

tokenizer_config.json

@@ -0,0 +1,185 @@
+{
+  "added_tokens_decoder": {
+    "199998": {
+      "content": "<|startoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "199999": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200000": {
+      "content": "<|reserved_200000|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200001": {
+      "content": "<|reserved_200001|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200002": {
+      "content": "<|return|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200003": {
+      "content": "<|constrain|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200004": {
+      "content": "<|reserved_200004|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200005": {
+      "content": "<|channel|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200006": {
+      "content": "<|start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200007": {
+      "content": "<|end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200008": {
+      "content": "<|message|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200009": {
+      "content": "<|reserved_200009|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200010": {
+      "content": "<|reserved_200010|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200011": {
+      "content": "<|reserved_200011|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200012": {
+      "content": "<|call|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200013": {
+      "content": "<|reserved_200013|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200014": {
+      "content": "<|reserved_200014|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200015": {
+      "content": "<|reserved_200015|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200016": {
+      "content": "<|reserved_200016|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200017": {
+      "content": "<|reserved_200017|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200018": {
+      "content": "<|endofprompt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|startoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|return|>",
+  "extra_special_tokens": {},
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|reserved_200017|>",
+  "padding_side": "right",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": null
+}

trainer_state.json

@@ -0,0 +1,2374 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.26485568760611206,
+  "eval_steps": 500,
+  "global_step": 156,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.001697792869269949,
+      "grad_norm": 253.41531372070312,
+      "learning_rate": 0.0,
+      "logits/chosen": -3.862927198410034,
+      "logits/rejected": -3.6351959705352783,
+      "logps/chosen": -605.5757446289062,
+      "logps/rejected": -1404.901611328125,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.003395585738539898,
+      "grad_norm": 168.63119506835938,
+      "learning_rate": 2e-07,
+      "logits/chosen": -3.7591066360473633,
+      "logits/rejected": -3.6339735984802246,
+      "logps/chosen": -829.5263061523438,
+      "logps/rejected": -1331.5780029296875,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 2
+    },
+    {
+      "epoch": 0.0050933786078098476,
+      "grad_norm": 250.63763427734375,
+      "learning_rate": 4e-07,
+      "logits/chosen": -3.887515068054199,
+      "logits/rejected": -3.7103145122528076,
+      "logps/chosen": -766.8009033203125,
+      "logps/rejected": -1616.171875,
+      "loss": 0.6411,
+      "rewards/accuracies": 0.6842105388641357,
+      "rewards/chosen": 0.6313945651054382,
+      "rewards/margins": 1.0291563272476196,
+      "rewards/rejected": -0.39776191115379333,
+      "step": 3
+    },
+    {
+      "epoch": 0.006791171477079796,
+      "grad_norm": 322.8396911621094,
+      "learning_rate": 6e-07,
+      "logits/chosen": -3.7688064575195312,
+      "logits/rejected": -3.654154062271118,
+      "logps/chosen": -627.8695068359375,
+      "logps/rejected": -1573.804443359375,
+      "loss": 1.1542,
+      "rewards/accuracies": 0.5789473652839661,
+      "rewards/chosen": 0.18179118633270264,
+      "rewards/margins": -0.08869992196559906,
+      "rewards/rejected": 0.2704910635948181,
+      "step": 4
+    },
+    {
+      "epoch": 0.008488964346349746,
+      "grad_norm": 421.36041259765625,
+      "learning_rate": 8e-07,
+      "logits/chosen": -3.8056693077087402,
+      "logits/rejected": -3.6669576168060303,
+      "logps/chosen": -917.1754760742188,
+      "logps/rejected": -1573.216552734375,
+      "loss": 1.5513,
+      "rewards/accuracies": 0.5263158082962036,
+      "rewards/chosen": -0.4203932285308838,
+      "rewards/margins": -0.8918607831001282,
+      "rewards/rejected": 0.4714674651622772,
+      "step": 5
+    },
+    {
+      "epoch": 0.010186757215619695,
+      "grad_norm": 465.82470703125,
+      "learning_rate": 1e-06,
+      "logits/chosen": -3.646392822265625,
+      "logits/rejected": -3.419647216796875,
+      "logps/chosen": -766.7083740234375,
+      "logps/rejected": -1917.0477294921875,
+      "loss": 1.0477,
+      "rewards/accuracies": 0.5789473652839661,
+      "rewards/chosen": 0.39575517177581787,
+      "rewards/margins": 0.6562828421592712,
+      "rewards/rejected": -0.2605276107788086,
+      "step": 6
+    },
+    {
+      "epoch": 0.011884550084889643,
+      "grad_norm": 383.790771484375,
+      "learning_rate": 1.2e-06,
+      "logits/chosen": -3.67680287361145,
+      "logits/rejected": -3.279794216156006,
+      "logps/chosen": -555.0008544921875,
+      "logps/rejected": -1844.1700439453125,
+      "loss": 1.1984,
+      "rewards/accuracies": 0.42105263471603394,
+      "rewards/chosen": 0.1972377598285675,
+      "rewards/margins": 0.061311546713113785,
+      "rewards/rejected": 0.1359262317419052,
+      "step": 7
+    },
+    {
+      "epoch": 0.013582342954159592,
+      "grad_norm": 193.729248046875,
+      "learning_rate": 1.4000000000000001e-06,
+      "logits/chosen": -3.8404300212860107,
+      "logits/rejected": -3.687358856201172,
+      "logps/chosen": -692.4404907226562,
+      "logps/rejected": -1474.986083984375,
+      "loss": 1.2099,
+      "rewards/accuracies": 0.4736842215061188,
+      "rewards/chosen": -0.3380126953125,
+      "rewards/margins": -0.10957071930170059,
+      "rewards/rejected": -0.2284420132637024,
+      "step": 8
+    },
+    {
+      "epoch": 0.015280135823429542,
+      "grad_norm": 295.02197265625,
+      "learning_rate": 1.6e-06,
+      "logits/chosen": -3.912959098815918,
+      "logits/rejected": -3.6805474758148193,
+      "logps/chosen": -551.9108276367188,
+      "logps/rejected": -1136.086181640625,
+      "loss": 2.2536,
+      "rewards/accuracies": 0.4736842215061188,
+      "rewards/chosen": -0.2959785461425781,
+      "rewards/margins": -1.3720390796661377,
+      "rewards/rejected": 1.0760605335235596,
+      "step": 9
+    },
+    {
+      "epoch": 0.01697792869269949,
+      "grad_norm": 257.409912109375,
+      "learning_rate": 1.7999999999999997e-06,
+      "logits/chosen": -3.78908634185791,
+      "logits/rejected": -3.679199457168579,
+      "logps/chosen": -439.6385803222656,
+      "logps/rejected": -1298.555908203125,
+      "loss": 1.1054,
+      "rewards/accuracies": 0.5263158082962036,
+      "rewards/chosen": 0.05673830211162567,
+      "rewards/margins": 0.06232653185725212,
+      "rewards/rejected": -0.005588180385529995,
+      "step": 10
+    },
+    {
+      "epoch": 0.01867572156196944,
+      "grad_norm": 293.6235656738281,
+      "learning_rate": 2e-06,
+      "logits/chosen": -3.7469851970672607,
+      "logits/rejected": -3.6060433387756348,
+      "logps/chosen": -759.355224609375,
+      "logps/rejected": -1455.1871337890625,
+      "loss": 1.2198,
+      "rewards/accuracies": 0.6315789222717285,
+      "rewards/chosen": 0.09574534744024277,
+      "rewards/margins": 0.17931774258613586,
+      "rewards/rejected": -0.08357242494821548,
+      "step": 11
+    },
+    {
+      "epoch": 0.02037351443123939,
+      "grad_norm": 298.3741760253906,
+      "learning_rate": 2.2e-06,
+      "logits/chosen": -3.8653757572174072,
+      "logits/rejected": -3.6336910724639893,
+      "logps/chosen": -719.357177734375,
+      "logps/rejected": -1460.9769287109375,
+      "loss": 1.3198,
+      "rewards/accuracies": 0.4736842215061188,
+      "rewards/chosen": -0.8121546506881714,
+      "rewards/margins": -0.4211699962615967,
+      "rewards/rejected": -0.3909846842288971,
+      "step": 12
+    },
+    {
+      "epoch": 0.022071307300509338,
+      "grad_norm": 560.5316772460938,
+      "learning_rate": 2.4e-06,
+      "logits/chosen": -3.7372677326202393,
+      "logits/rejected": -3.375757932662964,
+      "logps/chosen": -784.5247802734375,
+      "logps/rejected": -1696.49755859375,
+      "loss": 4.0524,
+      "rewards/accuracies": 0.31578946113586426,
+      "rewards/chosen": -0.8245162963867188,
+      "rewards/margins": -3.4090776443481445,
+      "rewards/rejected": 2.5845611095428467,
+      "step": 13
+    },
+    {
+      "epoch": 0.023769100169779286,
+      "grad_norm": 236.77005004882812,
+      "learning_rate": 2.6e-06,
+      "logits/chosen": -3.5540072917938232,
+      "logits/rejected": -3.5078113079071045,
+      "logps/chosen": -733.4524536132812,
+      "logps/rejected": -1333.154052734375,
+      "loss": 1.0773,
+      "rewards/accuracies": 0.6842105388641357,
+      "rewards/chosen": -0.5969735383987427,
+      "rewards/margins": 0.8910157084465027,
+      "rewards/rejected": -1.4879893064498901,
+      "step": 14
+    },
+    {
+      "epoch": 0.025466893039049237,
+      "grad_norm": 229.76531982421875,
+      "learning_rate": 2.8000000000000003e-06,
+      "logits/chosen": -3.9040117263793945,
+      "logits/rejected": -3.7774672508239746,
+      "logps/chosen": -635.8306884765625,
+      "logps/rejected": -1249.3895263671875,
+      "loss": 0.8262,
+      "rewards/accuracies": 0.8421052694320679,
+      "rewards/chosen": -0.1118985116481781,
+      "rewards/margins": 0.4985432028770447,
+      "rewards/rejected": -0.6104416847229004,
+      "step": 15
+    },
+    {
+      "epoch": 0.027164685908319185,
+      "grad_norm": 107.49576568603516,
+      "learning_rate": 2.9999999999999997e-06,
+      "logits/chosen": -3.6848840713500977,
+      "logits/rejected": -3.5065793991088867,
+      "logps/chosen": -634.2091674804688,
+      "logps/rejected": -1518.8646240234375,
+      "loss": 0.7195,
+      "rewards/accuracies": 0.6842105388641357,
+      "rewards/chosen": -0.577033519744873,
+      "rewards/margins": 1.5605580806732178,
+      "rewards/rejected": -2.137591600418091,
+      "step": 16
+    },
+    {
+      "epoch": 0.028862478777589132,
+      "grad_norm": 415.56146240234375,
+      "learning_rate": 3.2e-06,
+      "logits/chosen": -3.6984755992889404,
+      "logits/rejected": -3.463688850402832,
+      "logps/chosen": -764.843017578125,
+      "logps/rejected": -1857.59375,
+      "loss": 1.1264,
+      "rewards/accuracies": 0.6315789222717285,
+      "rewards/chosen": -0.6638931632041931,
+      "rewards/margins": 0.4968886077404022,
+      "rewards/rejected": -1.160781741142273,
+      "step": 17
+    },
+    {
+      "epoch": 0.030560271646859084,
+      "grad_norm": 94.64751434326172,
+      "learning_rate": 3.4e-06,
+      "logits/chosen": -3.751709222793579,
+      "logits/rejected": -3.501725912094116,
+      "logps/chosen": -1178.7249755859375,
+      "logps/rejected": -2049.426513671875,
+      "loss": 0.3738,
+      "rewards/accuracies": 0.7894737124443054,
+      "rewards/chosen": -0.984968900680542,
+      "rewards/margins": 2.2881321907043457,
+      "rewards/rejected": -3.2731010913848877,
+      "step": 18
+    },
+    {
+      "epoch": 0.03225806451612903,
+      "grad_norm": 189.4529266357422,
+      "learning_rate": 3.5999999999999994e-06,
+      "logits/chosen": -3.6436092853546143,
+      "logits/rejected": -3.5360822677612305,
+      "logps/chosen": -1026.79541015625,
+      "logps/rejected": -1805.1661376953125,
+      "loss": 0.6848,
+      "rewards/accuracies": 0.5789473652839661,
+      "rewards/chosen": -1.6188812255859375,
+      "rewards/margins": 1.1595208644866943,
+      "rewards/rejected": -2.778402090072632,
+      "step": 19
+    },
+    {
+      "epoch": 0.03395585738539898,
+      "grad_norm": 274.99066162109375,
+      "learning_rate": 3.7999999999999996e-06,
+      "logits/chosen": -3.6498701572418213,
+      "logits/rejected": -3.4178977012634277,
+      "logps/chosen": -999.5308227539062,
+      "logps/rejected": -2296.958251953125,
+      "loss": 0.6219,
+      "rewards/accuracies": 0.7368420958518982,
+      "rewards/chosen": -1.8587936162948608,
+      "rewards/margins": 1.8212978839874268,
+      "rewards/rejected": -3.680091381072998,
+      "step": 20
+    },
+    {
+      "epoch": 0.035653650254668934,
+      "grad_norm": 38.05210494995117,
+      "learning_rate": 4e-06,
+      "logits/chosen": -3.9445407390594482,
+      "logits/rejected": -3.6595799922943115,
+      "logps/chosen": -800.626220703125,
+      "logps/rejected": -1689.36474609375,
+      "loss": 0.3567,
+      "rewards/accuracies": 0.8421052694320679,
+      "rewards/chosen": -0.8077056407928467,
+      "rewards/margins": 3.5434539318084717,
+      "rewards/rejected": -4.351159572601318,
+      "step": 21
+    },
+    {
+      "epoch": 0.03735144312393888,
+      "grad_norm": 158.98568725585938,
+      "learning_rate": 4.2e-06,
+      "logits/chosen": -3.736311197280884,
+      "logits/rejected": -3.480236530303955,
+      "logps/chosen": -916.6001586914062,
+      "logps/rejected": -2222.215576171875,
+      "loss": 0.563,
+      "rewards/accuracies": 0.7368420958518982,
+      "rewards/chosen": -1.4374593496322632,
+      "rewards/margins": 4.035632610321045,
+      "rewards/rejected": -5.473092555999756,
+      "step": 22
+    },
+    {
+      "epoch": 0.03904923599320883,
+      "grad_norm": 70.41584777832031,
+      "learning_rate": 4.4e-06,
+      "logits/chosen": -3.833054780960083,
+      "logits/rejected": -3.6345179080963135,
+      "logps/chosen": -394.1758728027344,
+      "logps/rejected": -1003.4253540039062,
+      "loss": 0.4901,
+      "rewards/accuracies": 0.7894737124443054,
+      "rewards/chosen": -1.265379548072815,
+      "rewards/margins": 2.194890022277832,
+      "rewards/rejected": -3.4602696895599365,
+      "step": 23
+    },
+    {
+      "epoch": 0.04074702886247878,
+      "grad_norm": 421.5316162109375,
+      "learning_rate": 4.6e-06,
+      "logits/chosen": -3.865057945251465,
+      "logits/rejected": -3.7267401218414307,
+      "logps/chosen": -1011.3937377929688,
+      "logps/rejected": -1325.136474609375,
+      "loss": 2.5717,
+      "rewards/accuracies": 0.6315789222717285,
+      "rewards/chosen": -4.050071716308594,
+      "rewards/margins": 0.2841028869152069,
+      "rewards/rejected": -4.334174633026123,
+      "step": 24
+    },
+    {
+      "epoch": 0.042444821731748725,
+      "grad_norm": 23.352031707763672,
+      "learning_rate": 4.8e-06,
+      "logits/chosen": -3.747166395187378,
+      "logits/rejected": -3.5740578174591064,
+      "logps/chosen": -636.3488159179688,
+      "logps/rejected": -1744.6492919921875,
+      "loss": 0.2064,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.844959020614624,
+      "rewards/margins": 4.199265956878662,
+      "rewards/rejected": -6.044224739074707,
+      "step": 25
+    },
+    {
+      "epoch": 0.044142614601018676,
+      "grad_norm": 195.68930053710938,
+      "learning_rate": 5e-06,
+      "logits/chosen": -3.7053744792938232,
+      "logits/rejected": -3.458592176437378,
+      "logps/chosen": -1098.926513671875,
+      "logps/rejected": -2076.791015625,
+      "loss": 0.8933,
+      "rewards/accuracies": 0.7368420958518982,
+      "rewards/chosen": -4.474255084991455,
+      "rewards/margins": 6.412208080291748,
+      "rewards/rejected": -10.886463165283203,
+      "step": 26
+    },
+    {
+      "epoch": 0.04584040747028863,
+      "grad_norm": 227.4545440673828,
+      "learning_rate": 5.2e-06,
+      "logits/chosen": -3.8032331466674805,
+      "logits/rejected": -3.699617862701416,
+      "logps/chosen": -1081.37841796875,
+      "logps/rejected": -1176.47314453125,
+      "loss": 2.6358,
+      "rewards/accuracies": 0.6315789222717285,
+      "rewards/chosen": -4.119551181793213,
+      "rewards/margins": 1.8443418741226196,
+      "rewards/rejected": -5.963892459869385,
+      "step": 27
+    },
+    {
+      "epoch": 0.04753820033955857,
+      "grad_norm": 34.18882751464844,
+      "learning_rate": 5.4e-06,
+      "logits/chosen": -3.7299857139587402,
+      "logits/rejected": -3.5211801528930664,
+      "logps/chosen": -769.3715209960938,
+      "logps/rejected": -2031.3714599609375,
+      "loss": 0.3846,
+      "rewards/accuracies": 0.8421052694320679,
+      "rewards/chosen": -3.4353513717651367,
+      "rewards/margins": 7.42952299118042,
+      "rewards/rejected": -10.864873886108398,
+      "step": 28
+    },
+    {
+      "epoch": 0.04923599320882852,
+      "grad_norm": 47.27467727661133,
+      "learning_rate": 5.600000000000001e-06,
+      "logits/chosen": -3.7908668518066406,
+      "logits/rejected": -3.70328950881958,
+      "logps/chosen": -450.0216979980469,
+      "logps/rejected": -850.3040161132812,
+      "loss": 0.3451,
+      "rewards/accuracies": 0.8421052694320679,
+      "rewards/chosen": -2.090538501739502,
+      "rewards/margins": 2.738893747329712,
+      "rewards/rejected": -4.829432010650635,
+      "step": 29
+    },
+    {
+      "epoch": 0.050933786078098474,
+      "grad_norm": 57.626739501953125,
+      "learning_rate": 5.8e-06,
+      "logits/chosen": -3.8924810886383057,
+      "logits/rejected": -3.5905206203460693,
+      "logps/chosen": -835.1837158203125,
+      "logps/rejected": -1747.001220703125,
+      "loss": 0.4026,
+      "rewards/accuracies": 0.8947368264198303,
+      "rewards/chosen": -4.321366786956787,
+      "rewards/margins": 6.867539882659912,
+      "rewards/rejected": -11.1889066696167,
+      "step": 30
+    },
+    {
+      "epoch": 0.05263157894736842,
+      "grad_norm": 38.01590347290039,
+      "learning_rate": 5.999999999999999e-06,
+      "logits/chosen": -3.851223945617676,
+      "logits/rejected": -3.510930299758911,
+      "logps/chosen": -1061.4251708984375,
+      "logps/rejected": -2431.38818359375,
+      "loss": 0.2778,
+      "rewards/accuracies": 0.8421052694320679,
+      "rewards/chosen": -4.868941783905029,
+      "rewards/margins": 8.6685209274292,
+      "rewards/rejected": -13.537463188171387,
+      "step": 31
+    },
+    {
+      "epoch": 0.05432937181663837,
+      "grad_norm": 212.42942810058594,
+      "learning_rate": 6.2e-06,
+      "logits/chosen": -3.774933099746704,
+      "logits/rejected": -3.769662857055664,
+      "logps/chosen": -772.5845336914062,
+      "logps/rejected": -1347.6492919921875,
+      "loss": 1.2241,
+      "rewards/accuracies": 0.7894737124443054,
+      "rewards/chosen": -4.480807304382324,
+      "rewards/margins": 4.54947566986084,
+      "rewards/rejected": -9.030282020568848,
+      "step": 32
+    },
+    {
+      "epoch": 0.05602716468590832,
+      "grad_norm": 19.666946411132812,
+      "learning_rate": 6.4e-06,
+      "logits/chosen": -3.743957281112671,
+      "logits/rejected": -3.4936163425445557,
+      "logps/chosen": -682.9749145507812,
+      "logps/rejected": -1827.3819580078125,
+      "loss": 0.1635,
+      "rewards/accuracies": 0.8947368264198303,
+      "rewards/chosen": -3.1712920665740967,
+      "rewards/margins": 6.201633930206299,
+      "rewards/rejected": -9.372926712036133,
+      "step": 33
+    },
+    {
+      "epoch": 0.057724957555178265,
+      "grad_norm": 120.70894622802734,
+      "learning_rate": 6.5999999999999995e-06,
+      "logits/chosen": -3.7753212451934814,
+      "logits/rejected": -3.556863784790039,
+      "logps/chosen": -671.4615478515625,
+      "logps/rejected": -1785.2418212890625,
+      "loss": 0.318,
+      "rewards/accuracies": 0.8947368264198303,
+      "rewards/chosen": -3.6064252853393555,
+      "rewards/margins": 8.610679626464844,
+      "rewards/rejected": -12.2171049118042,
+      "step": 34
+    },
+    {
+      "epoch": 0.059422750424448216,
+      "grad_norm": 142.81478881835938,
+      "learning_rate": 6.8e-06,
+      "logits/chosen": -3.6816179752349854,
+      "logits/rejected": -3.592928647994995,
+      "logps/chosen": -983.4131469726562,
+      "logps/rejected": -1972.7388916015625,
+      "loss": 0.741,
+      "rewards/accuracies": 0.8421052694320679,
+      "rewards/chosen": -4.976840496063232,
+      "rewards/margins": 7.173476219177246,
+      "rewards/rejected": -12.150318145751953,
+      "step": 35
+    },
+    {
+      "epoch": 0.06112054329371817,
+      "grad_norm": 55.45768356323242,
+      "learning_rate": 7e-06,
+      "logits/chosen": -3.8779475688934326,
+      "logits/rejected": -3.7793450355529785,
+      "logps/chosen": -339.6081848144531,
+      "logps/rejected": -951.3660888671875,
+      "loss": 0.2939,
+      "rewards/accuracies": 0.8947368264198303,
+      "rewards/chosen": -1.9719674587249756,
+      "rewards/margins": 3.296804189682007,
+      "rewards/rejected": -5.268771648406982,
+      "step": 36
+    },
+    {
+      "epoch": 0.06281833616298811,
+      "grad_norm": 30.784549713134766,
+      "learning_rate": 6.999995681795924e-06,
+      "logits/chosen": -3.828057050704956,
+      "logits/rejected": -3.502441644668579,
+      "logps/chosen": -714.7034301757812,
+      "logps/rejected": -2708.83349609375,
+      "loss": 0.2977,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": -3.3775370121002197,
+      "rewards/margins": 11.722590446472168,
+      "rewards/rejected": -15.100126266479492,
+      "step": 37
+    },
+    {
+      "epoch": 0.06451612903225806,
+      "grad_norm": 92.5975341796875,
+      "learning_rate": 6.999982727197902e-06,
+      "logits/chosen": -3.8347432613372803,
+      "logits/rejected": -3.7575533390045166,
+      "logps/chosen": -636.1466674804688,
+      "logps/rejected": -1029.7694091796875,
+      "loss": 0.7757,
+      "rewards/accuracies": 0.8421052694320679,
+      "rewards/chosen": -3.5677731037139893,
+      "rewards/margins": 1.545227289199829,
+      "rewards/rejected": -5.113000392913818,
+      "step": 38
+    },
+    {
+      "epoch": 0.06621392190152801,
+      "grad_norm": 95.85789489746094,
+      "learning_rate": 6.999961136248556e-06,
+      "logits/chosen": -4.125431060791016,
+      "logits/rejected": -3.8893556594848633,
+      "logps/chosen": -410.7164306640625,
+      "logps/rejected": -891.2525024414062,
+      "loss": 0.6768,
+      "rewards/accuracies": 0.7894737124443054,
+      "rewards/chosen": -2.504324197769165,
+      "rewards/margins": 3.1136813163757324,
+      "rewards/rejected": -5.618005275726318,
+      "step": 39
+    },
+    {
+      "epoch": 0.06791171477079797,
+      "grad_norm": 100.32817840576172,
+      "learning_rate": 6.999930909018923e-06,
+      "logits/chosen": -3.7379164695739746,
+      "logits/rejected": -3.6506543159484863,
+      "logps/chosen": -773.4288940429688,
+      "logps/rejected": -1535.619384765625,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.7894737124443054,
+      "rewards/chosen": -4.3702168464660645,
+      "rewards/margins": 5.706230163574219,
+      "rewards/rejected": -10.076446533203125,
+      "step": 40
+    },
+    {
+      "epoch": 0.06960950764006792,
+      "grad_norm": 265.2519226074219,
+      "learning_rate": 6.999892045608449e-06,
+      "logits/chosen": -3.7866342067718506,
+      "logits/rejected": -3.6257829666137695,
+      "logps/chosen": -1463.1141357421875,
+      "logps/rejected": -1968.306396484375,
+      "loss": 1.7624,
+      "rewards/accuracies": 0.7368420958518982,
+      "rewards/chosen": -8.517228126525879,
+      "rewards/margins": 3.0944013595581055,
+      "rewards/rejected": -11.611628532409668,
+      "step": 41
+    },
+    {
+      "epoch": 0.07130730050933787,
+      "grad_norm": 172.8343048095703,
+      "learning_rate": 6.999844546145e-06,
+      "logits/chosen": -3.7290871143341064,
+      "logits/rejected": -3.678785800933838,
+      "logps/chosen": -1073.1202392578125,
+      "logps/rejected": -1557.5167236328125,
+      "loss": 1.1741,
+      "rewards/accuracies": 0.6315789222717285,
+      "rewards/chosen": -4.971875190734863,
+      "rewards/margins": 5.183844566345215,
+      "rewards/rejected": -10.155719757080078,
+      "step": 42
+    },
+    {
+      "epoch": 0.0730050933786078,
+      "grad_norm": 439.6719665527344,
+      "learning_rate": 6.999788410784851e-06,
+      "logits/chosen": -3.736490249633789,
+      "logits/rejected": -3.7374424934387207,
+      "logps/chosen": -970.9537353515625,
+      "logps/rejected": -1044.634521484375,
+      "loss": 2.492,
+      "rewards/accuracies": 0.7368420958518982,
+      "rewards/chosen": -4.464043617248535,
+      "rewards/margins": 1.9694724082946777,
+      "rewards/rejected": -6.433516025543213,
+      "step": 43
+    },
+    {
+      "epoch": 0.07470288624787776,
+      "grad_norm": 348.20208740234375,
+      "learning_rate": 6.999723639712691e-06,
+      "logits/chosen": -3.7370145320892334,
+      "logits/rejected": -3.5357182025909424,
+      "logps/chosen": -1153.1453857421875,
+      "logps/rejected": -2073.501220703125,
+      "loss": 1.224,
+      "rewards/accuracies": 0.8947368264198303,
+      "rewards/chosen": -5.5407490730285645,
+      "rewards/margins": 6.521337032318115,
+      "rewards/rejected": -12.06208610534668,
+      "step": 44
+    },
+    {
+      "epoch": 0.07640067911714771,
+      "grad_norm": 88.37775421142578,
+      "learning_rate": 6.99965023314162e-06,
+      "logits/chosen": -3.579968214035034,
+      "logits/rejected": -3.5226123332977295,
+      "logps/chosen": -890.927734375,
+      "logps/rejected": -1632.11865234375,
+      "loss": 0.4036,
+      "rewards/accuracies": 0.8947368264198303,
+      "rewards/chosen": -3.7555582523345947,
+      "rewards/margins": 5.139606952667236,
+      "rewards/rejected": -8.89516544342041,
+      "step": 45
+    },
+    {
+      "epoch": 0.07809847198641766,
+      "grad_norm": 26.1304988861084,
+      "learning_rate": 6.999568191313151e-06,
+      "logits/chosen": -3.8651726245880127,
+      "logits/rejected": -3.720208168029785,
+      "logps/chosen": -614.7899169921875,
+      "logps/rejected": -1408.1048583984375,
+      "loss": 0.1974,
+      "rewards/accuracies": 0.8947368264198303,
+      "rewards/chosen": -1.4795647859573364,
+      "rewards/margins": 4.5847673416137695,
+      "rewards/rejected": -6.064332008361816,
+      "step": 46
+    },
+    {
+      "epoch": 0.07979626485568761,
+      "grad_norm": 113.91500091552734,
+      "learning_rate": 6.999477514497206e-06,
+      "logits/chosen": -3.823136568069458,
+      "logits/rejected": -3.4229936599731445,
+      "logps/chosen": -528.0443115234375,
+      "logps/rejected": -1226.7037353515625,
+      "loss": 0.7325,
+      "rewards/accuracies": 0.5263158082962036,
+      "rewards/chosen": -2.1668002605438232,
+      "rewards/margins": 3.469679117202759,
+      "rewards/rejected": -5.63647985458374,
+      "step": 47
+    },
+    {
+      "epoch": 0.08149405772495756,
+      "grad_norm": 489.7789611816406,
+      "learning_rate": 6.999378202992117e-06,
+      "logits/chosen": -3.5525624752044678,
+      "logits/rejected": -3.5467617511749268,
+      "logps/chosen": -1388.6363525390625,
+      "logps/rejected": -2001.3951416015625,
+      "loss": 1.8689,
+      "rewards/accuracies": 0.7368420958518982,
+      "rewards/chosen": -4.708498477935791,
+      "rewards/margins": 5.626183032989502,
+      "rewards/rejected": -10.334681510925293,
+      "step": 48
+    },
+    {
+      "epoch": 0.0831918505942275,
+      "grad_norm": 22.89203453063965,
+      "learning_rate": 6.999270257124625e-06,
+      "logits/chosen": -3.796808958053589,
+      "logits/rejected": -3.505606174468994,
+      "logps/chosen": -892.1825561523438,
+      "logps/rejected": -2162.55908203125,
+      "loss": 0.163,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": -2.251508951187134,
+      "rewards/margins": 7.360548973083496,
+      "rewards/rejected": -9.612058639526367,
+      "step": 49
+    },
+    {
+      "epoch": 0.08488964346349745,
+      "grad_norm": 91.42052459716797,
+      "learning_rate": 6.999153677249879e-06,
+      "logits/chosen": -3.908195972442627,
+      "logits/rejected": -3.6327476501464844,
+      "logps/chosen": -673.4215087890625,
+      "logps/rejected": -1590.754150390625,
+      "loss": 0.4303,
+      "rewards/accuracies": 0.7894737124443054,
+      "rewards/chosen": -1.6777422428131104,
+      "rewards/margins": 4.665722370147705,
+      "rewards/rejected": -6.3434648513793945,
+      "step": 50
+    },
+    {
+      "epoch": 0.0865874363327674,
+      "grad_norm": 91.4797134399414,
+      "learning_rate": 6.999028463751432e-06,
+      "logits/chosen": -3.7286806106567383,
+      "logits/rejected": -3.413952112197876,
+      "logps/chosen": -520.708984375,
+      "logps/rejected": -1529.3165283203125,
+      "loss": 0.5809,
+      "rewards/accuracies": 0.7894737124443054,
+      "rewards/chosen": -1.926355242729187,
+      "rewards/margins": 4.901339054107666,
+      "rewards/rejected": -6.827694416046143,
+      "step": 51
+    },
+    {
+      "epoch": 0.08828522920203735,
+      "grad_norm": 212.3939208984375,
+      "learning_rate": 6.998894617041247e-06,
+      "logits/chosen": -3.603062868118286,
+      "logits/rejected": -3.29253888130188,
+      "logps/chosen": -808.3012084960938,
+      "logps/rejected": -1734.44677734375,
+      "loss": 0.4248,
+      "rewards/accuracies": 0.8947368264198303,
+      "rewards/chosen": -2.2691493034362793,
+      "rewards/margins": 3.9479000568389893,
+      "rewards/rejected": -6.217049598693848,
+      "step": 52
+    },
+    {
+      "epoch": 0.0899830220713073,
+      "grad_norm": 74.79203033447266,
+      "learning_rate": 6.998752137559685e-06,
+      "logits/chosen": -3.6790192127227783,
+      "logits/rejected": -3.5017669200897217,
+      "logps/chosen": -953.6409301757812,
+      "logps/rejected": -1656.254150390625,
+      "loss": 0.4278,
+      "rewards/accuracies": 0.7894737124443054,
+      "rewards/chosen": -1.8970955610275269,
+      "rewards/margins": 4.207927703857422,
+      "rewards/rejected": -6.10502290725708,
+      "step": 53
+    },
+    {
+      "epoch": 0.09168081494057725,
+      "grad_norm": 59.26031494140625,
+      "learning_rate": 6.998601025775514e-06,
+      "logits/chosen": -3.8182411193847656,
+      "logits/rejected": -3.5321524143218994,
+      "logps/chosen": -736.5953979492188,
+      "logps/rejected": -1425.5853271484375,
+      "loss": 0.264,
+      "rewards/accuracies": 0.8421052694320679,
+      "rewards/chosen": -1.5467214584350586,
+      "rewards/margins": 4.007129669189453,
+      "rewards/rejected": -5.553851127624512,
+      "step": 54
+    },
+    {
+      "epoch": 0.0933786078098472,
+      "grad_norm": 52.41440200805664,
+      "learning_rate": 6.9984412821858986e-06,
+      "logits/chosen": -3.9926223754882812,
+      "logits/rejected": -3.7583560943603516,
+      "logps/chosen": -829.6281127929688,
+      "logps/rejected": -1729.273681640625,
+      "loss": 0.2221,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": -0.7696541547775269,
+      "rewards/margins": 6.237390041351318,
+      "rewards/rejected": -7.007043838500977,
+      "step": 55
+    },
+    {
+      "epoch": 0.09507640067911714,
+      "grad_norm": 26.63044548034668,
+      "learning_rate": 6.998272907316408e-06,
+      "logits/chosen": -3.779285430908203,
+      "logits/rejected": -3.7332136631011963,
+      "logps/chosen": -813.6694946289062,
+      "logps/rejected": -1219.005126953125,
+      "loss": 0.2965,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": -0.7820217609405518,
+      "rewards/margins": 3.99786376953125,
+      "rewards/rejected": -4.779885768890381,
+      "step": 56
+    },
+    {
+      "epoch": 0.0967741935483871,
+      "grad_norm": 200.0863037109375,
+      "learning_rate": 6.998095901721002e-06,
+      "logits/chosen": -3.737215042114258,
+      "logits/rejected": -3.4522533416748047,
+      "logps/chosen": -1219.6558837890625,
+      "logps/rejected": -2281.621337890625,
+      "loss": 0.6528,
+      "rewards/accuracies": 0.7368420958518982,
+      "rewards/chosen": -1.4408289194107056,
+      "rewards/margins": 5.970677852630615,
+      "rewards/rejected": -7.4115071296691895,
+      "step": 57
+    },
+    {
+      "epoch": 0.09847198641765705,
+      "grad_norm": 117.3527603149414,
+      "learning_rate": 6.997910265982044e-06,
+      "logits/chosen": -3.8976874351501465,
+      "logits/rejected": -3.5120694637298584,
+      "logps/chosen": -645.6788940429688,
+      "logps/rejected": -2199.60107421875,
+      "loss": 0.3866,
+      "rewards/accuracies": 0.8947368264198303,
+      "rewards/chosen": -1.1234122514724731,
+      "rewards/margins": 9.132166862487793,
+      "rewards/rejected": -10.25557804107666,
+      "step": 58
+    },
+    {
+      "epoch": 0.100169779286927,
+      "grad_norm": 102.26219177246094,
+      "learning_rate": 6.997716000710282e-06,
+      "logits/chosen": -3.730203866958618,
+      "logits/rejected": -3.469224452972412,
+      "logps/chosen": -665.0298461914062,
+      "logps/rejected": -2010.6336669921875,
+      "loss": 0.5415,
+      "rewards/accuracies": 0.8947368264198303,
+      "rewards/chosen": -1.3938777446746826,
+      "rewards/margins": 6.654629230499268,
+      "rewards/rejected": -8.048505783081055,
+      "step": 59
+    },
+    {
+      "epoch": 0.10186757215619695,
+      "grad_norm": 61.66415786743164,
+      "learning_rate": 6.9975131065448635e-06,
+      "logits/chosen": -3.8243424892425537,
+      "logits/rejected": -3.732562780380249,
+      "logps/chosen": -607.7645263671875,
+      "logps/rejected": -1131.4736328125,
+      "loss": 0.5046,
+      "rewards/accuracies": 0.8947368264198303,
+      "rewards/chosen": -1.3792232275009155,
+      "rewards/margins": 3.7461965084075928,
+      "rewards/rejected": -5.125420093536377,
+      "step": 60
+    },
+    {
+      "epoch": 0.1035653650254669,
+      "grad_norm": 37.84029769897461,
+      "learning_rate": 6.997301584153322e-06,
+      "logits/chosen": -3.714604377746582,
+      "logits/rejected": -3.463052749633789,
+      "logps/chosen": -762.448486328125,
+      "logps/rejected": -2047.9002685546875,
+      "loss": 0.1614,
+      "rewards/accuracies": 0.8947368264198303,
+      "rewards/chosen": -1.0677417516708374,
+      "rewards/margins": 8.98747730255127,
+      "rewards/rejected": -10.055219650268555,
+      "step": 61
+    },
+    {
+      "epoch": 0.10526315789473684,
+      "grad_norm": 82.76973724365234,
+      "learning_rate": 6.9970814342315785e-06,
+      "logits/chosen": -3.89551043510437,
+      "logits/rejected": -3.806321859359741,
+      "logps/chosen": -1028.5377197265625,
+      "logps/rejected": -1512.9459228515625,
+      "loss": 0.3038,
+      "rewards/accuracies": 0.8947368264198303,
+      "rewards/chosen": -1.632301926612854,
+      "rewards/margins": 6.467922210693359,
+      "rewards/rejected": -8.100224494934082,
+      "step": 62
+    },
+    {
+      "epoch": 0.10696095076400679,
+      "grad_norm": 64.66484069824219,
+      "learning_rate": 6.99685265750394e-06,
+      "logits/chosen": -3.8749654293060303,
+      "logits/rejected": -3.5141024589538574,
+      "logps/chosen": -803.72265625,
+      "logps/rejected": -2069.438720703125,
+      "loss": 0.149,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": -0.3676658570766449,
+      "rewards/margins": 6.284032344818115,
+      "rewards/rejected": -6.651698589324951,
+      "step": 63
+    },
+    {
+      "epoch": 0.10865874363327674,
+      "grad_norm": 319.979736328125,
+      "learning_rate": 6.996615254723095e-06,
+      "logits/chosen": -3.723146915435791,
+      "logits/rejected": -3.653900623321533,
+      "logps/chosen": -1207.6829833984375,
+      "logps/rejected": -1694.005859375,
+      "loss": 1.8275,
+      "rewards/accuracies": 0.6315789222717285,
+      "rewards/chosen": -1.6885120868682861,
+      "rewards/margins": 1.9671132564544678,
+      "rewards/rejected": -3.655625104904175,
+      "step": 64
+    },
+    {
+      "epoch": 0.11035653650254669,
+      "grad_norm": 36.157405853271484,
+      "learning_rate": 6.996369226670114e-06,
+      "logits/chosen": -3.987626552581787,
+      "logits/rejected": -3.628830909729004,
+      "logps/chosen": -341.0587463378906,
+      "logps/rejected": -1282.7177734375,
+      "loss": 0.2223,
+      "rewards/accuracies": 0.8421052694320679,
+      "rewards/chosen": -1.118723750114441,
+      "rewards/margins": 4.808956146240234,
+      "rewards/rejected": -5.927680015563965,
+      "step": 65
+    },
+    {
+      "epoch": 0.11205432937181664,
+      "grad_norm": 22.443660736083984,
+      "learning_rate": 6.996114574154442e-06,
+      "logits/chosen": -3.928772211074829,
+      "logits/rejected": -3.497762680053711,
+      "logps/chosen": -930.9664916992188,
+      "logps/rejected": -2155.483154296875,
+      "loss": 0.1819,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": -0.1744617372751236,
+      "rewards/margins": 8.700950622558594,
+      "rewards/rejected": -8.875412940979004,
+      "step": 66
+    },
+    {
+      "epoch": 0.11375212224108659,
+      "grad_norm": 130.13873291015625,
+      "learning_rate": 6.9958512980139036e-06,
+      "logits/chosen": -3.6982197761535645,
+      "logits/rejected": -3.524240016937256,
+      "logps/chosen": -549.5940551757812,
+      "logps/rejected": -1785.2578125,
+      "loss": 0.7641,
+      "rewards/accuracies": 0.8421052694320679,
+      "rewards/chosen": -1.5123597383499146,
+      "rewards/margins": 3.1788170337677,
+      "rewards/rejected": -4.691176414489746,
+      "step": 67
+    },
+    {
+      "epoch": 0.11544991511035653,
+      "grad_norm": 51.48603820800781,
+      "learning_rate": 6.995579399114692e-06,
+      "logits/chosen": -3.842341423034668,
+      "logits/rejected": -3.610034942626953,
+      "logps/chosen": -806.690673828125,
+      "logps/rejected": -2141.96630859375,
+      "loss": 0.2012,
+      "rewards/accuracies": 0.8947368264198303,
+      "rewards/chosen": -1.566497802734375,
+      "rewards/margins": 7.737715721130371,
+      "rewards/rejected": -9.304213523864746,
+      "step": 68
+    },
+    {
+      "epoch": 0.11714770797962648,
+      "grad_norm": 73.42926025390625,
+      "learning_rate": 6.995298878351372e-06,
+      "logits/chosen": -3.790009021759033,
+      "logits/rejected": -3.6310887336730957,
+      "logps/chosen": -944.5301513671875,
+      "logps/rejected": -1037.1728515625,
+      "loss": 0.602,
+      "rewards/accuracies": 0.7894737124443054,
+      "rewards/chosen": -1.2610541582107544,
+      "rewards/margins": 3.599123477935791,
+      "rewards/rejected": -4.860177516937256,
+      "step": 69
+    },
+    {
+      "epoch": 0.11884550084889643,
+      "grad_norm": 21.031574249267578,
+      "learning_rate": 6.995009736646872e-06,
+      "logits/chosen": -3.8047678470611572,
+      "logits/rejected": -3.6022684574127197,
+      "logps/chosen": -574.5223388671875,
+      "logps/rejected": -1691.0224609375,
+      "loss": 0.121,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": -1.6642251014709473,
+      "rewards/margins": 8.040745735168457,
+      "rewards/rejected": -9.704970359802246,
+      "step": 70
+    },
+    {
+      "epoch": 0.12054329371816638,
+      "grad_norm": 16.911624908447266,
+      "learning_rate": 6.9947119749524874e-06,
+      "logits/chosen": -3.818938732147217,
+      "logits/rejected": -3.619452953338623,
+      "logps/chosen": -481.9064636230469,
+      "logps/rejected": -1320.2601318359375,
+      "loss": 0.1583,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6912651062011719,
+      "rewards/margins": 8.028682708740234,
+      "rewards/rejected": -8.71994686126709,
+      "step": 71
+    },
+    {
+      "epoch": 0.12224108658743633,
+      "grad_norm": 12.058719635009766,
+      "learning_rate": 6.994405594247871e-06,
+      "logits/chosen": -3.6893556118011475,
+      "logits/rejected": -3.5537164211273193,
+      "logps/chosen": -683.3974609375,
+      "logps/rejected": -1866.5242919921875,
+      "loss": 0.1022,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6975005865097046,
+      "rewards/margins": 10.15715217590332,
+      "rewards/rejected": -10.854653358459473,
+      "step": 72
+    },
+    {
+      "epoch": 0.12393887945670629,
+      "grad_norm": 22.33544921875,
+      "learning_rate": 6.9940905955410325e-06,
+      "logits/chosen": -3.5932538509368896,
+      "logits/rejected": -3.4058444499969482,
+      "logps/chosen": -1016.451904296875,
+      "logps/rejected": -2503.58642578125,
+      "loss": 0.1588,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": -1.3289124965667725,
+      "rewards/margins": 11.663505554199219,
+      "rewards/rejected": -12.99241828918457,
+      "step": 73
+    },
+    {
+      "epoch": 0.12563667232597622,
+      "grad_norm": 22.422624588012695,
+      "learning_rate": 6.99376697986834e-06,
+      "logits/chosen": -3.8296735286712646,
+      "logits/rejected": -3.6365044116973877,
+      "logps/chosen": -1009.485595703125,
+      "logps/rejected": -1367.0968017578125,
+      "loss": 0.164,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8882247805595398,
+      "rewards/margins": 7.45314359664917,
+      "rewards/rejected": -8.341367721557617,
+      "step": 74
+    },
+    {
+      "epoch": 0.1273344651952462,
+      "grad_norm": 49.275638580322266,
+      "learning_rate": 6.993434748294506e-06,
+      "logits/chosen": -3.7765982151031494,
+      "logits/rejected": -3.6348676681518555,
+      "logps/chosen": -1144.813720703125,
+      "logps/rejected": -1617.056884765625,
+      "loss": 0.255,
+      "rewards/accuracies": 0.8947368264198303,
+      "rewards/chosen": -0.4225502014160156,
+      "rewards/margins": 9.486503601074219,
+      "rewards/rejected": -9.909053802490234,
+      "step": 75
+    },
+    {
+      "epoch": 0.12903225806451613,
+      "grad_norm": 14.187338829040527,
+      "learning_rate": 6.993093901912594e-06,
+      "logits/chosen": -3.6928181648254395,
+      "logits/rejected": -3.587698221206665,
+      "logps/chosen": -553.68505859375,
+      "logps/rejected": -1374.8743896484375,
+      "loss": 0.0957,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.0303900241851807,
+      "rewards/margins": 7.220921516418457,
+      "rewards/rejected": -8.251312255859375,
+      "step": 76
+    },
+    {
+      "epoch": 0.1307300509337861,
+      "grad_norm": 14.263720512390137,
+      "learning_rate": 6.992744441844007e-06,
+      "logits/chosen": -3.684743881225586,
+      "logits/rejected": -3.4999022483825684,
+      "logps/chosen": -932.639404296875,
+      "logps/rejected": -1908.8145751953125,
+      "loss": 0.1539,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.9391114711761475,
+      "rewards/margins": 9.202366828918457,
+      "rewards/rejected": -12.141478538513184,
+      "step": 77
+    },
+    {
+      "epoch": 0.13242784380305603,
+      "grad_norm": 37.87944412231445,
+      "learning_rate": 6.992386369238492e-06,
+      "logits/chosen": -3.895080327987671,
+      "logits/rejected": -3.6478140354156494,
+      "logps/chosen": -429.8042907714844,
+      "logps/rejected": -1550.5736083984375,
+      "loss": 0.3366,
+      "rewards/accuracies": 0.8421052694320679,
+      "rewards/chosen": -1.025573492050171,
+      "rewards/margins": 10.461471557617188,
+      "rewards/rejected": -11.487046241760254,
+      "step": 78
+    },
+    {
+      "epoch": 0.13412563667232597,
+      "grad_norm": 10.795161247253418,
+      "learning_rate": 6.992019685274128e-06,
+      "logits/chosen": -3.830293655395508,
+      "logits/rejected": -3.716765880584717,
+      "logps/chosen": -636.1162719726562,
+      "logps/rejected": -1580.7022705078125,
+      "loss": 0.1227,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4873403310775757,
+      "rewards/margins": 9.842334747314453,
+      "rewards/rejected": -10.32967472076416,
+      "step": 79
+    },
+    {
+      "epoch": 0.13582342954159593,
+      "grad_norm": 19.715940475463867,
+      "learning_rate": 6.991644391157327e-06,
+      "logits/chosen": -3.8085625171661377,
+      "logits/rejected": -3.702777147293091,
+      "logps/chosen": -763.5365600585938,
+      "logps/rejected": -1607.0970458984375,
+      "loss": 0.1805,
+      "rewards/accuracies": 0.8947368264198303,
+      "rewards/chosen": -2.4620907306671143,
+      "rewards/margins": 7.387575626373291,
+      "rewards/rejected": -9.849666595458984,
+      "step": 80
+    },
+    {
+      "epoch": 0.13752122241086587,
+      "grad_norm": 147.5608673095703,
+      "learning_rate": 6.991260488122832e-06,
+      "logits/chosen": -3.74634051322937,
+      "logits/rejected": -3.639721155166626,
+      "logps/chosen": -759.9743041992188,
+      "logps/rejected": -1637.945556640625,
+      "loss": 0.2248,
+      "rewards/accuracies": 0.8947368264198303,
+      "rewards/chosen": -2.1698291301727295,
+      "rewards/margins": 7.532597541809082,
+      "rewards/rejected": -9.70242691040039,
+      "step": 81
+    },
+    {
+      "epoch": 0.13921901528013583,
+      "grad_norm": 7.893219947814941,
+      "learning_rate": 6.990867977433703e-06,
+      "logits/chosen": -3.686493158340454,
+      "logits/rejected": -3.47658109664917,
+      "logps/chosen": -626.2376708984375,
+      "logps/rejected": -1273.48046875,
+      "loss": 0.078,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.2047574520111084,
+      "rewards/margins": 7.101351261138916,
+      "rewards/rejected": -8.306108474731445,
+      "step": 82
+    },
+    {
+      "epoch": 0.14091680814940577,
+      "grad_norm": 275.6655578613281,
+      "learning_rate": 6.9904668603813265e-06,
+      "logits/chosen": -3.8455450534820557,
+      "logits/rejected": -3.6755669116973877,
+      "logps/chosen": -638.1837768554688,
+      "logps/rejected": -1698.2481689453125,
+      "loss": 0.6272,
+      "rewards/accuracies": 0.8421052694320679,
+      "rewards/chosen": -2.9386696815490723,
+      "rewards/margins": 6.407962322235107,
+      "rewards/rejected": -9.34663200378418,
+      "step": 83
+    },
+    {
+      "epoch": 0.14261460101867574,
+      "grad_norm": 9.246347427368164,
+      "learning_rate": 6.990057138285402e-06,
+      "logits/chosen": -3.7783308029174805,
+      "logits/rejected": -3.5027096271514893,
+      "logps/chosen": -459.2795715332031,
+      "logps/rejected": -1837.5609130859375,
+      "loss": 0.1398,
+      "rewards/accuracies": 0.8947368264198303,
+      "rewards/chosen": -0.7564946413040161,
+      "rewards/margins": 8.159191131591797,
+      "rewards/rejected": -8.91568660736084,
+      "step": 84
+    },
+    {
+      "epoch": 0.14431239388794567,
+      "grad_norm": 26.28587532043457,
+      "learning_rate": 6.9896388124939385e-06,
+      "logits/chosen": -3.699047088623047,
+      "logits/rejected": -3.5644023418426514,
+      "logps/chosen": -845.7753295898438,
+      "logps/rejected": -1895.5594482421875,
+      "loss": 0.1398,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": -3.2373931407928467,
+      "rewards/margins": 8.989819526672363,
+      "rewards/rejected": -12.227212905883789,
+      "step": 85
+    },
+    {
+      "epoch": 0.1460101867572156,
+      "grad_norm": 49.27679443359375,
+      "learning_rate": 6.989211884383254e-06,
+      "logits/chosen": -3.911393165588379,
+      "logits/rejected": -3.6924586296081543,
+      "logps/chosen": -981.2985229492188,
+      "logps/rejected": -1688.3194580078125,
+      "loss": 0.368,
+      "rewards/accuracies": 0.7894737124443054,
+      "rewards/chosen": -1.8452825546264648,
+      "rewards/margins": 7.300060749053955,
+      "rewards/rejected": -9.145342826843262,
+      "step": 86
+    },
+    {
+      "epoch": 0.14770797962648557,
+      "grad_norm": 16.920671463012695,
+      "learning_rate": 6.988776355357967e-06,
+      "logits/chosen": -3.7513556480407715,
+      "logits/rejected": -3.6575586795806885,
+      "logps/chosen": -398.22503662109375,
+      "logps/rejected": -1223.1280517578125,
+      "loss": 0.1006,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": -1.5688518285751343,
+      "rewards/margins": 7.068317890167236,
+      "rewards/rejected": -8.637168884277344,
+      "step": 87
+    },
+    {
+      "epoch": 0.1494057724957555,
+      "grad_norm": 81.00189208984375,
+      "learning_rate": 6.988332226850995e-06,
+      "logits/chosen": -3.869905471801758,
+      "logits/rejected": -3.6356797218322754,
+      "logps/chosen": -724.948974609375,
+      "logps/rejected": -1465.9176025390625,
+      "loss": 0.2425,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": -3.615725517272949,
+      "rewards/margins": 7.304844856262207,
+      "rewards/rejected": -10.920570373535156,
+      "step": 88
+    },
+    {
+      "epoch": 0.15110356536502548,
+      "grad_norm": 50.97134780883789,
+      "learning_rate": 6.987879500323547e-06,
+      "logits/chosen": -3.8716299533843994,
+      "logits/rejected": -3.752835512161255,
+      "logps/chosen": -795.7817993164062,
+      "logps/rejected": -1463.76953125,
+      "loss": 0.2313,
+      "rewards/accuracies": 0.8421052694320679,
+      "rewards/chosen": -1.9299911260604858,
+      "rewards/margins": 5.114964008331299,
+      "rewards/rejected": -7.044955730438232,
+      "step": 89
+    },
+    {
+      "epoch": 0.15280135823429541,
+      "grad_norm": 17.053251266479492,
+      "learning_rate": 6.987418177265119e-06,
+      "logits/chosen": -3.694180965423584,
+      "logits/rejected": -3.3970630168914795,
+      "logps/chosen": -637.548095703125,
+      "logps/rejected": -1742.90771484375,
+      "loss": 0.111,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": -1.3107919692993164,
+      "rewards/margins": 11.130751609802246,
+      "rewards/rejected": -12.441544532775879,
+      "step": 90
+    },
+    {
+      "epoch": 0.15449915110356535,
+      "grad_norm": 28.220325469970703,
+      "learning_rate": 6.986948259193496e-06,
+      "logits/chosen": -3.5280370712280273,
+      "logits/rejected": -3.5956943035125732,
+      "logps/chosen": -671.525634765625,
+      "logps/rejected": -1569.584228515625,
+      "loss": 0.11,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": -2.0381264686584473,
+      "rewards/margins": 6.511880874633789,
+      "rewards/rejected": -8.550007820129395,
+      "step": 91
+    },
+    {
+      "epoch": 0.15619694397283532,
+      "grad_norm": 158.85935974121094,
+      "learning_rate": 6.986469747654732e-06,
+      "logits/chosen": -3.6342899799346924,
+      "logits/rejected": -3.681292772293091,
+      "logps/chosen": -1075.5013427734375,
+      "logps/rejected": -988.7138061523438,
+      "loss": 1.2643,
+      "rewards/accuracies": 0.8421052694320679,
+      "rewards/chosen": -2.830941915512085,
+      "rewards/margins": 4.036745071411133,
+      "rewards/rejected": -6.867687225341797,
+      "step": 92
+    },
+    {
+      "epoch": 0.15789473684210525,
+      "grad_norm": 3.5202670097351074,
+      "learning_rate": 6.9859826442231625e-06,
+      "logits/chosen": -3.8045427799224854,
+      "logits/rejected": -3.614744186401367,
+      "logps/chosen": -1360.7662353515625,
+      "logps/rejected": -2518.14013671875,
+      "loss": 0.051,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8611124753952026,
+      "rewards/margins": 13.58449649810791,
+      "rewards/rejected": -14.445610046386719,
+      "step": 93
+    },
+    {
+      "epoch": 0.15959252971137522,
+      "grad_norm": 66.50271606445312,
+      "learning_rate": 6.985486950501385e-06,
+      "logits/chosen": -3.8674557209014893,
+      "logits/rejected": -3.7710487842559814,
+      "logps/chosen": -451.79132080078125,
+      "logps/rejected": -1597.1959228515625,
+      "loss": 0.0991,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": -1.6133522987365723,
+      "rewards/margins": 7.80655574798584,
+      "rewards/rejected": -9.419907569885254,
+      "step": 94
+    },
+    {
+      "epoch": 0.16129032258064516,
+      "grad_norm": 6.91108512878418,
+      "learning_rate": 6.984982668120262e-06,
+      "logits/chosen": -3.7838284969329834,
+      "logits/rejected": -3.4987387657165527,
+      "logps/chosen": -527.23974609375,
+      "logps/rejected": -1747.5931396484375,
+      "loss": 0.0727,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.144969940185547,
+      "rewards/margins": 12.582115173339844,
+      "rewards/rejected": -14.727084159851074,
+      "step": 95
+    },
+    {
+      "epoch": 0.16298811544991512,
+      "grad_norm": 10.717843055725098,
+      "learning_rate": 6.984469798738912e-06,
+      "logits/chosen": -3.8322417736053467,
+      "logits/rejected": -3.60394549369812,
+      "logps/chosen": -1170.5518798828125,
+      "logps/rejected": -2239.024169921875,
+      "loss": 0.1157,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.399217128753662,
+      "rewards/margins": 10.41158390045166,
+      "rewards/rejected": -12.81080150604248,
+      "step": 96
+    },
+    {
+      "epoch": 0.16468590831918506,
+      "grad_norm": 107.81643676757812,
+      "learning_rate": 6.983948344044709e-06,
+      "logits/chosen": -3.8501274585723877,
+      "logits/rejected": -3.6387856006622314,
+      "logps/chosen": -872.6104125976562,
+      "logps/rejected": -1869.1826171875,
+      "loss": 0.2127,
+      "rewards/accuracies": 0.8947368264198303,
+      "rewards/chosen": -2.3860466480255127,
+      "rewards/margins": 8.92795181274414,
+      "rewards/rejected": -11.313998222351074,
+      "step": 97
+    },
+    {
+      "epoch": 0.166383701188455,
+      "grad_norm": 73.27976989746094,
+      "learning_rate": 6.983418305753268e-06,
+      "logits/chosen": -3.683082342147827,
+      "logits/rejected": -3.383187770843506,
+      "logps/chosen": -760.8966674804688,
+      "logps/rejected": -1680.0477294921875,
+      "loss": 0.149,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": -2.4082255363464355,
+      "rewards/margins": 11.904454231262207,
+      "rewards/rejected": -14.3126802444458,
+      "step": 98
+    },
+    {
+      "epoch": 0.16808149405772496,
+      "grad_norm": 44.875701904296875,
+      "learning_rate": 6.982879685608448e-06,
+      "logits/chosen": -3.912052631378174,
+      "logits/rejected": -3.6544899940490723,
+      "logps/chosen": -673.0807495117188,
+      "logps/rejected": -1691.139892578125,
+      "loss": 0.2513,
+      "rewards/accuracies": 0.8947368264198303,
+      "rewards/chosen": -2.335156202316284,
+      "rewards/margins": 11.730042457580566,
+      "rewards/rejected": -14.06519889831543,
+      "step": 99
+    },
+    {
+      "epoch": 0.1697792869269949,
+      "grad_norm": 71.96469116210938,
+      "learning_rate": 6.9823324853823406e-06,
+      "logits/chosen": -3.76344633102417,
+      "logits/rejected": -3.4823975563049316,
+      "logps/chosen": -821.2760009765625,
+      "logps/rejected": -1709.872802734375,
+      "loss": 0.2505,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": -1.0189168453216553,
+      "rewards/margins": 11.905284881591797,
+      "rewards/rejected": -12.924201965332031,
+      "step": 100
+    },
+    {
+      "epoch": 0.17147707979626486,
+      "grad_norm": 59.67416763305664,
+      "learning_rate": 6.981776706875268e-06,
+      "logits/chosen": -3.8769843578338623,
+      "logits/rejected": -3.453199863433838,
+      "logps/chosen": -772.9574584960938,
+      "logps/rejected": -1973.6256103515625,
+      "loss": 0.2506,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": -2.354810953140259,
+      "rewards/margins": 9.986964225769043,
+      "rewards/rejected": -12.341774940490723,
+      "step": 101
+    },
+    {
+      "epoch": 0.1731748726655348,
+      "grad_norm": 37.598140716552734,
+      "learning_rate": 6.981212351915777e-06,
+      "logits/chosen": -3.723578453063965,
+      "logits/rejected": -3.579558849334717,
+      "logps/chosen": -740.1079711914062,
+      "logps/rejected": -1366.8663330078125,
+      "loss": 0.1841,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": -0.11994344741106033,
+      "rewards/margins": 8.2774658203125,
+      "rewards/rejected": -8.397409439086914,
+      "step": 102
+    },
+    {
+      "epoch": 0.17487266553480477,
+      "grad_norm": 35.35015869140625,
+      "learning_rate": 6.980639422360626e-06,
+      "logits/chosen": -3.8196682929992676,
+      "logits/rejected": -3.6046993732452393,
+      "logps/chosen": -948.5782470703125,
+      "logps/rejected": -2096.224853515625,
+      "loss": 0.1762,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": -1.7156829833984375,
+      "rewards/margins": 16.56784439086914,
+      "rewards/rejected": -18.283527374267578,
+      "step": 103
+    },
+    {
+      "epoch": 0.1765704584040747,
+      "grad_norm": 186.90672302246094,
+      "learning_rate": 6.98005792009479e-06,
+      "logits/chosen": -3.76750111579895,
+      "logits/rejected": -3.532073497772217,
+      "logps/chosen": -979.2677001953125,
+      "logps/rejected": -2216.670166015625,
+      "loss": 0.8309,
+      "rewards/accuracies": 0.8947368264198303,
+      "rewards/chosen": -1.9807201623916626,
+      "rewards/margins": 15.777000427246094,
+      "rewards/rejected": -17.757719039916992,
+      "step": 104
+    },
+    {
+      "epoch": 0.17826825127334464,
+      "grad_norm": 6.775128364562988,
+      "learning_rate": 6.979467847031445e-06,
+      "logits/chosen": -3.8047983646392822,
+      "logits/rejected": -3.674755096435547,
+      "logps/chosen": -651.359619140625,
+      "logps/rejected": -1356.9620361328125,
+      "loss": 0.0426,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.110494613647461,
+      "rewards/margins": 7.81674861907959,
+      "rewards/rejected": -9.927244186401367,
+      "step": 105
+    },
+    {
+      "epoch": 0.1799660441426146,
+      "grad_norm": 8.4512357711792,
+      "learning_rate": 6.978869205111969e-06,
+      "logits/chosen": -3.6785566806793213,
+      "logits/rejected": -3.462512969970703,
+      "logps/chosen": -730.62109375,
+      "logps/rejected": -2025.4356689453125,
+      "loss": 0.1019,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -3.5104362964630127,
+      "rewards/margins": 13.898301124572754,
+      "rewards/rejected": -17.408737182617188,
+      "step": 106
+    },
+    {
+      "epoch": 0.18166383701188454,
+      "grad_norm": 12.687128067016602,
+      "learning_rate": 6.978261996305929e-06,
+      "logits/chosen": -3.799273729324341,
+      "logits/rejected": -3.5843749046325684,
+      "logps/chosen": -661.6332397460938,
+      "logps/rejected": -1790.1246337890625,
+      "loss": 0.0524,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.5980563163757324,
+      "rewards/margins": 13.51810359954834,
+      "rewards/rejected": -16.116159439086914,
+      "step": 107
+    },
+    {
+      "epoch": 0.1833616298811545,
+      "grad_norm": 106.98979187011719,
+      "learning_rate": 6.977646222611078e-06,
+      "logits/chosen": -3.8738813400268555,
+      "logits/rejected": -3.7034285068511963,
+      "logps/chosen": -989.0670166015625,
+      "logps/rejected": -1602.0150146484375,
+      "loss": 0.4934,
+      "rewards/accuracies": 0.8421052694320679,
+      "rewards/chosen": -4.357776641845703,
+      "rewards/margins": 12.145195960998535,
+      "rewards/rejected": -16.502975463867188,
+      "step": 108
+    },
+    {
+      "epoch": 0.18505942275042445,
+      "grad_norm": 16.131244659423828,
+      "learning_rate": 6.977021886053347e-06,
+      "logits/chosen": -3.9964237213134766,
+      "logits/rejected": -3.872692108154297,
+      "logps/chosen": -809.6708984375,
+      "logps/rejected": -1341.9288330078125,
+      "loss": 0.1253,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": -3.4534506797790527,
+      "rewards/margins": 8.198371887207031,
+      "rewards/rejected": -11.651823043823242,
+      "step": 109
+    },
+    {
+      "epoch": 0.1867572156196944,
+      "grad_norm": 12.353798866271973,
+      "learning_rate": 6.976388988686845e-06,
+      "logits/chosen": -3.726100206375122,
+      "logits/rejected": -3.5875403881073,
+      "logps/chosen": -528.83642578125,
+      "logps/rejected": -1781.2496337890625,
+      "loss": 0.0947,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": -2.2456202507019043,
+      "rewards/margins": 10.44008731842041,
+      "rewards/rejected": -12.685708045959473,
+      "step": 110
+    },
+    {
+      "epoch": 0.18845500848896435,
+      "grad_norm": 2.563150644302368,
+      "learning_rate": 6.975747532593838e-06,
+      "logits/chosen": -3.5097496509552,
+      "logits/rejected": -3.4688761234283447,
+      "logps/chosen": -395.9211120605469,
+      "logps/rejected": -1319.6453857421875,
+      "loss": 0.0244,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.6569132804870605,
+      "rewards/margins": 12.773691177368164,
+      "rewards/rejected": -15.430604934692383,
+      "step": 111
+    },
+    {
+      "epoch": 0.19015280135823429,
+      "grad_norm": 3.8946115970611572,
+      "learning_rate": 6.975097519884759e-06,
+      "logits/chosen": -3.901890277862549,
+      "logits/rejected": -3.7657508850097656,
+      "logps/chosen": -673.4298706054688,
+      "logps/rejected": -1513.614501953125,
+      "loss": 0.0335,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -4.2903852462768555,
+      "rewards/margins": 12.002580642700195,
+      "rewards/rejected": -16.292964935302734,
+      "step": 112
+    },
+    {
+      "epoch": 0.19185059422750425,
+      "grad_norm": 16.817495346069336,
+      "learning_rate": 6.974438952698188e-06,
+      "logits/chosen": -3.7428741455078125,
+      "logits/rejected": -3.5564656257629395,
+      "logps/chosen": -801.8799438476562,
+      "logps/rejected": -1992.5880126953125,
+      "loss": 0.103,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": -4.0022053718566895,
+      "rewards/margins": 24.54104232788086,
+      "rewards/rejected": -28.543249130249023,
+      "step": 113
+    },
+    {
+      "epoch": 0.1935483870967742,
+      "grad_norm": 163.4203338623047,
+      "learning_rate": 6.973771833200849e-06,
+      "logits/chosen": -3.662956714630127,
+      "logits/rejected": -3.532480478286743,
+      "logps/chosen": -744.2583618164062,
+      "logps/rejected": -1503.3968505859375,
+      "loss": 0.9508,
+      "rewards/accuracies": 0.7894737124443054,
+      "rewards/chosen": -4.864002227783203,
+      "rewards/margins": 12.667509078979492,
+      "rewards/rejected": -17.531509399414062,
+      "step": 114
+    },
+    {
+      "epoch": 0.19524617996604415,
+      "grad_norm": 38.810176849365234,
+      "learning_rate": 6.973096163587607e-06,
+      "logits/chosen": -4.001194000244141,
+      "logits/rejected": -3.762303590774536,
+      "logps/chosen": -988.1875,
+      "logps/rejected": -2089.39794921875,
+      "loss": 0.142,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": -6.069306373596191,
+      "rewards/margins": 13.245327949523926,
+      "rewards/rejected": -19.314632415771484,
+      "step": 115
+    },
+    {
+      "epoch": 0.1969439728353141,
+      "grad_norm": 158.87086486816406,
+      "learning_rate": 6.972411946081454e-06,
+      "logits/chosen": -3.7854738235473633,
+      "logits/rejected": -3.7265615463256836,
+      "logps/chosen": -971.9598388671875,
+      "logps/rejected": -1453.19921875,
+      "loss": 0.8761,
+      "rewards/accuracies": 0.8421052694320679,
+      "rewards/chosen": -6.6403303146362305,
+      "rewards/margins": 8.726883888244629,
+      "rewards/rejected": -15.367213249206543,
+      "step": 116
+    },
+    {
+      "epoch": 0.19864176570458403,
+      "grad_norm": 20.066919326782227,
+      "learning_rate": 6.97171918293351e-06,
+      "logits/chosen": -3.973656415939331,
+      "logits/rejected": -3.67142915725708,
+      "logps/chosen": -573.3673706054688,
+      "logps/rejected": -1267.1904296875,
+      "loss": 0.288,
+      "rewards/accuracies": 0.8947368264198303,
+      "rewards/chosen": -3.8687334060668945,
+      "rewards/margins": 10.996365547180176,
+      "rewards/rejected": -14.865097999572754,
+      "step": 117
+    },
+    {
+      "epoch": 0.200339558573854,
+      "grad_norm": 65.05223083496094,
+      "learning_rate": 6.971017876423005e-06,
+      "logits/chosen": -3.750948905944824,
+      "logits/rejected": -3.4453625679016113,
+      "logps/chosen": -574.1520385742188,
+      "logps/rejected": -1289.5355224609375,
+      "loss": 0.6536,
+      "rewards/accuracies": 0.7894737124443054,
+      "rewards/chosen": -3.2062273025512695,
+      "rewards/margins": 11.94129753112793,
+      "rewards/rejected": -15.1475248336792,
+      "step": 118
+    },
+    {
+      "epoch": 0.20203735144312393,
+      "grad_norm": 4.754962921142578,
+      "learning_rate": 6.970308028857279e-06,
+      "logits/chosen": -3.821983814239502,
+      "logits/rejected": -3.5136260986328125,
+      "logps/chosen": -224.42532348632812,
+      "logps/rejected": -1365.0841064453125,
+      "loss": 0.0363,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.837965965270996,
+      "rewards/margins": 15.45004940032959,
+      "rewards/rejected": -17.288015365600586,
+      "step": 119
+    },
+    {
+      "epoch": 0.2037351443123939,
+      "grad_norm": 131.49501037597656,
+      "learning_rate": 6.9695896425717765e-06,
+      "logits/chosen": -3.7603652477264404,
+      "logits/rejected": -3.7311620712280273,
+      "logps/chosen": -855.5682373046875,
+      "logps/rejected": -1257.74462890625,
+      "loss": 1.2102,
+      "rewards/accuracies": 0.7894737124443054,
+      "rewards/chosen": -5.832756996154785,
+      "rewards/margins": 5.7323317527771,
+      "rewards/rejected": -11.565089225769043,
+      "step": 120
+    },
+    {
+      "epoch": 0.20543293718166383,
+      "grad_norm": 70.33955383300781,
+      "learning_rate": 6.968862719930029e-06,
+      "logits/chosen": -3.762054204940796,
+      "logits/rejected": -3.4738166332244873,
+      "logps/chosen": -1078.19775390625,
+      "logps/rejected": -2073.80810546875,
+      "loss": 0.4222,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": -5.891961574554443,
+      "rewards/margins": 15.96943473815918,
+      "rewards/rejected": -21.86139488220215,
+      "step": 121
+    },
+    {
+      "epoch": 0.2071307300509338,
+      "grad_norm": 5.440072059631348,
+      "learning_rate": 6.968127263323654e-06,
+      "logits/chosen": -3.7950057983398438,
+      "logits/rejected": -3.603194236755371,
+      "logps/chosen": -640.3604736328125,
+      "logps/rejected": -1276.7269287109375,
+      "loss": 0.1369,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": -2.9976754188537598,
+      "rewards/margins": 11.999473571777344,
+      "rewards/rejected": -14.997147560119629,
+      "step": 122
+    },
+    {
+      "epoch": 0.20882852292020374,
+      "grad_norm": 147.45884704589844,
+      "learning_rate": 6.9673832751723526e-06,
+      "logits/chosen": -3.827544927597046,
+      "logits/rejected": -3.5036275386810303,
+      "logps/chosen": -1263.84619140625,
+      "logps/rejected": -2409.511962890625,
+      "loss": 0.3158,
+      "rewards/accuracies": 0.8947368264198303,
+      "rewards/chosen": -4.569932460784912,
+      "rewards/margins": 22.8966007232666,
+      "rewards/rejected": -27.466533660888672,
+      "step": 123
+    },
+    {
+      "epoch": 0.21052631578947367,
+      "grad_norm": 18.83106803894043,
+      "learning_rate": 6.9666307579238885e-06,
+      "logits/chosen": -3.645846366882324,
+      "logits/rejected": -3.535503387451172,
+      "logps/chosen": -837.787841796875,
+      "logps/rejected": -2416.427734375,
+      "loss": 0.1439,
+      "rewards/accuracies": 0.8947368264198303,
+      "rewards/chosen": -3.7407400608062744,
+      "rewards/margins": 20.752798080444336,
+      "rewards/rejected": -24.4935359954834,
+      "step": 124
+    },
+    {
+      "epoch": 0.21222410865874364,
+      "grad_norm": 35.632877349853516,
+      "learning_rate": 6.965869714054089e-06,
+      "logits/chosen": -3.700308322906494,
+      "logits/rejected": -3.5264322757720947,
+      "logps/chosen": -1051.30419921875,
+      "logps/rejected": -2074.9658203125,
+      "loss": 0.4841,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": -3.4254095554351807,
+      "rewards/margins": 16.705263137817383,
+      "rewards/rejected": -20.130672454833984,
+      "step": 125
+    },
+    {
+      "epoch": 0.21392190152801357,
+      "grad_norm": 21.05853271484375,
+      "learning_rate": 6.9651001460668325e-06,
+      "logits/chosen": -3.7296669483184814,
+      "logits/rejected": -3.6159603595733643,
+      "logps/chosen": -1007.9923095703125,
+      "logps/rejected": -1943.0440673828125,
+      "loss": 0.079,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": -2.7694876194000244,
+      "rewards/margins": 17.527170181274414,
+      "rewards/rejected": -20.29665756225586,
+      "step": 126
+    },
+    {
+      "epoch": 0.21561969439728354,
+      "grad_norm": 76.85245513916016,
+      "learning_rate": 6.964322056494046e-06,
+      "logits/chosen": -3.7412173748016357,
+      "logits/rejected": -3.6870882511138916,
+      "logps/chosen": -1203.6776123046875,
+      "logps/rejected": -1530.6995849609375,
+      "loss": 0.5236,
+      "rewards/accuracies": 0.8947368264198303,
+      "rewards/chosen": -4.34645414352417,
+      "rewards/margins": 7.942860126495361,
+      "rewards/rejected": -12.289313316345215,
+      "step": 127
+    },
+    {
+      "epoch": 0.21731748726655348,
+      "grad_norm": 23.3140869140625,
+      "learning_rate": 6.963535447895692e-06,
+      "logits/chosen": -3.8947396278381348,
+      "logits/rejected": -3.4670283794403076,
+      "logps/chosen": -579.88427734375,
+      "logps/rejected": -1736.227783203125,
+      "loss": 0.07,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": -1.4946991205215454,
+      "rewards/margins": 13.216540336608887,
+      "rewards/rejected": -14.711237907409668,
+      "step": 128
+    },
+    {
+      "epoch": 0.21901528013582344,
+      "grad_norm": 136.1709747314453,
+      "learning_rate": 6.962740322859758e-06,
+      "logits/chosen": -3.870145320892334,
+      "logits/rejected": -3.882458209991455,
+      "logps/chosen": -903.0145263671875,
+      "logps/rejected": -1209.391845703125,
+      "loss": 0.9088,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": -2.7953643798828125,
+      "rewards/margins": 5.496224403381348,
+      "rewards/rejected": -8.29158878326416,
+      "step": 129
+    },
+    {
+      "epoch": 0.22071307300509338,
+      "grad_norm": 11.993281364440918,
+      "learning_rate": 6.961936684002254e-06,
+      "logits/chosen": -3.721262216567993,
+      "logits/rejected": -3.5086491107940674,
+      "logps/chosen": -547.963623046875,
+      "logps/rejected": -1809.1829833984375,
+      "loss": 0.1129,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5284523963928223,
+      "rewards/margins": 13.97046184539795,
+      "rewards/rejected": -14.49891471862793,
+      "step": 130
+    },
+    {
+      "epoch": 0.22241086587436332,
+      "grad_norm": 140.55343627929688,
+      "learning_rate": 6.9611245339672e-06,
+      "logits/chosen": -3.625781774520874,
+      "logits/rejected": -3.551302909851074,
+      "logps/chosen": -1080.9185791015625,
+      "logps/rejected": -1491.4063720703125,
+      "loss": 0.5151,
+      "rewards/accuracies": 0.7894737124443054,
+      "rewards/chosen": 0.7861904501914978,
+      "rewards/margins": 16.398347854614258,
+      "rewards/rejected": -15.612156867980957,
+      "step": 131
+    },
+    {
+      "epoch": 0.22410865874363328,
+      "grad_norm": 11.50153636932373,
+      "learning_rate": 6.960303875426619e-06,
+      "logits/chosen": -3.79368257522583,
+      "logits/rejected": -3.450730323791504,
+      "logps/chosen": -650.6856689453125,
+      "logps/rejected": -2077.446044921875,
+      "loss": 0.0191,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.2040356546640396,
+      "rewards/margins": 16.944364547729492,
+      "rewards/rejected": -17.148399353027344,
+      "step": 132
+    },
+    {
+      "epoch": 0.22580645161290322,
+      "grad_norm": 19.209768295288086,
+      "learning_rate": 6.9594747110805286e-06,
+      "logits/chosen": -3.7598936557769775,
+      "logits/rejected": -3.5663001537323,
+      "logps/chosen": -866.5333862304688,
+      "logps/rejected": -2260.24560546875,
+      "loss": 0.1021,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": 0.8966318368911743,
+      "rewards/margins": 20.567066192626953,
+      "rewards/rejected": -19.670434951782227,
+      "step": 133
+    },
+    {
+      "epoch": 0.22750424448217318,
+      "grad_norm": 55.371131896972656,
+      "learning_rate": 6.958637043656925e-06,
+      "logits/chosen": -3.8568215370178223,
+      "logits/rejected": -3.6581625938415527,
+      "logps/chosen": -405.2693176269531,
+      "logps/rejected": -1300.62255859375,
+      "loss": 0.4807,
+      "rewards/accuracies": 0.8421052694320679,
+      "rewards/chosen": -1.189803957939148,
+      "rewards/margins": 9.788188934326172,
+      "rewards/rejected": -10.97799301147461,
+      "step": 134
+    },
+    {
+      "epoch": 0.22920203735144312,
+      "grad_norm": 10.552238464355469,
+      "learning_rate": 6.95779087591179e-06,
+      "logits/chosen": -3.7003872394561768,
+      "logits/rejected": -3.5009264945983887,
+      "logps/chosen": -957.4447021484375,
+      "logps/rejected": -1280.665283203125,
+      "loss": 0.1362,
+      "rewards/accuracies": 0.8947368264198303,
+      "rewards/chosen": 2.2291970252990723,
+      "rewards/margins": 12.491341590881348,
+      "rewards/rejected": -10.2621431350708,
+      "step": 135
+    },
+    {
+      "epoch": 0.23089983022071306,
+      "grad_norm": 166.77572631835938,
+      "learning_rate": 6.9569362106290635e-06,
+      "logits/chosen": -3.6627299785614014,
+      "logits/rejected": -3.443359375,
+      "logps/chosen": -830.742919921875,
+      "logps/rejected": -1653.124755859375,
+      "loss": 0.8788,
+      "rewards/accuracies": 0.8947368264198303,
+      "rewards/chosen": 0.2320985347032547,
+      "rewards/margins": 9.47817325592041,
+      "rewards/rejected": -9.246074676513672,
+      "step": 136
+    },
+    {
+      "epoch": 0.23259762308998302,
+      "grad_norm": 9.919652938842773,
+      "learning_rate": 6.956073050620645e-06,
+      "logits/chosen": -3.8356752395629883,
+      "logits/rejected": -3.5876097679138184,
+      "logps/chosen": -756.435302734375,
+      "logps/rejected": -1525.454833984375,
+      "loss": 0.0908,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": 1.2842128276824951,
+      "rewards/margins": 14.17322063446045,
+      "rewards/rejected": -12.889008522033691,
+      "step": 137
+    },
+    {
+      "epoch": 0.23429541595925296,
+      "grad_norm": 12.168522834777832,
+      "learning_rate": 6.955201398726383e-06,
+      "logits/chosen": -3.819178342819214,
+      "logits/rejected": -3.7008984088897705,
+      "logps/chosen": -615.0360717773438,
+      "logps/rejected": -1958.1883544921875,
+      "loss": 0.0804,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.900735080242157,
+      "rewards/margins": 14.141839027404785,
+      "rewards/rejected": -13.241105079650879,
+      "step": 138
+    },
+    {
+      "epoch": 0.23599320882852293,
+      "grad_norm": 14.561647415161133,
+      "learning_rate": 6.954321257814067e-06,
+      "logits/chosen": -3.8233957290649414,
+      "logits/rejected": -3.609365224838257,
+      "logps/chosen": -722.4047241210938,
+      "logps/rejected": -1712.92578125,
+      "loss": 0.0818,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": -1.3475664854049683,
+      "rewards/margins": 9.34595012664795,
+      "rewards/rejected": -10.693516731262207,
+      "step": 139
+    },
+    {
+      "epoch": 0.23769100169779286,
+      "grad_norm": 5.579987525939941,
+      "learning_rate": 6.953432630779414e-06,
+      "logits/chosen": -3.9780023097991943,
+      "logits/rejected": -3.562074661254883,
+      "logps/chosen": -511.3399353027344,
+      "logps/rejected": -1746.2120361328125,
+      "loss": 0.0969,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.11427301913499832,
+      "rewards/margins": 15.565413475036621,
+      "rewards/rejected": -15.6796875,
+      "step": 140
+    },
+    {
+      "epoch": 0.23938879456706283,
+      "grad_norm": 4.00973653793335,
+      "learning_rate": 6.952535520546056e-06,
+      "logits/chosen": -3.7256813049316406,
+      "logits/rejected": -3.7113306522369385,
+      "logps/chosen": -1653.21337890625,
+      "logps/rejected": -1951.7298583984375,
+      "loss": 0.0419,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.7850377559661865,
+      "rewards/margins": 12.975248336791992,
+      "rewards/rejected": -11.190210342407227,
+      "step": 141
+    },
+    {
+      "epoch": 0.24108658743633277,
+      "grad_norm": 3.7720530033111572,
+      "learning_rate": 6.951629930065545e-06,
+      "logits/chosen": -3.5269370079040527,
+      "logits/rejected": -3.300715208053589,
+      "logps/chosen": -980.2633666992188,
+      "logps/rejected": -1817.9522705078125,
+      "loss": 0.1722,
+      "rewards/accuracies": 0.8947368264198303,
+      "rewards/chosen": 1.4748613834381104,
+      "rewards/margins": 16.97820472717285,
+      "rewards/rejected": -15.503344535827637,
+      "step": 142
+    },
+    {
+      "epoch": 0.2427843803056027,
+      "grad_norm": 69.65367126464844,
+      "learning_rate": 6.9507158623173235e-06,
+      "logits/chosen": -3.771528482437134,
+      "logits/rejected": -3.7239933013916016,
+      "logps/chosen": -451.2080078125,
+      "logps/rejected": -1115.76220703125,
+      "loss": 0.13,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": -0.8256826996803284,
+      "rewards/margins": 6.603112697601318,
+      "rewards/rejected": -7.428795337677002,
+      "step": 143
+    },
+    {
+      "epoch": 0.24448217317487267,
+      "grad_norm": 9.254491806030273,
+      "learning_rate": 6.949793320308733e-06,
+      "logits/chosen": -3.796405792236328,
+      "logits/rejected": -3.6394333839416504,
+      "logps/chosen": -495.36358642578125,
+      "logps/rejected": -1412.7320556640625,
+      "loss": 0.0838,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": 0.09402646869421005,
+      "rewards/margins": 11.091241836547852,
+      "rewards/rejected": -10.997215270996094,
+      "step": 144
+    },
+    {
+      "epoch": 0.2461799660441426,
+      "grad_norm": 118.1439208984375,
+      "learning_rate": 6.9488623070749915e-06,
+      "logits/chosen": -3.6636123657226562,
+      "logits/rejected": -3.4513723850250244,
+      "logps/chosen": -871.6858520507812,
+      "logps/rejected": -1940.8848876953125,
+      "loss": 0.2337,
+      "rewards/accuracies": 0.8947368264198303,
+      "rewards/chosen": -1.4755420684814453,
+      "rewards/margins": 12.448883056640625,
+      "rewards/rejected": -13.924426078796387,
+      "step": 145
+    },
+    {
+      "epoch": 0.24787775891341257,
+      "grad_norm": 25.487436294555664,
+      "learning_rate": 6.9479228256791875e-06,
+      "logits/chosen": -3.828705072402954,
+      "logits/rejected": -3.668996572494507,
+      "logps/chosen": -972.7256469726562,
+      "logps/rejected": -1655.1583251953125,
+      "loss": 0.1623,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": 0.741838276386261,
+      "rewards/margins": 10.798667907714844,
+      "rewards/rejected": -10.056829452514648,
+      "step": 146
+    },
+    {
+      "epoch": 0.2495755517826825,
+      "grad_norm": 8.61157512664795,
+      "learning_rate": 6.946974879212271e-06,
+      "logits/chosen": -3.916123628616333,
+      "logits/rejected": -3.7406952381134033,
+      "logps/chosen": -967.9044189453125,
+      "logps/rejected": -1524.1856689453125,
+      "loss": 0.1081,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": -0.8007361888885498,
+      "rewards/margins": 9.817014694213867,
+      "rewards/rejected": -10.617751121520996,
+      "step": 147
+    },
+    {
+      "epoch": 0.25127334465195245,
+      "grad_norm": 19.947677612304688,
+      "learning_rate": 6.946018470793046e-06,
+      "logits/chosen": -3.9741430282592773,
+      "logits/rejected": -3.751094341278076,
+      "logps/chosen": -494.927734375,
+      "logps/rejected": -1189.2178955078125,
+      "loss": 0.1311,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": -0.5863356590270996,
+      "rewards/margins": 11.22431755065918,
+      "rewards/rejected": -11.810651779174805,
+      "step": 148
+    },
+    {
+      "epoch": 0.2529711375212224,
+      "grad_norm": 5.324405670166016,
+      "learning_rate": 6.945053603568151e-06,
+      "logits/chosen": -3.7551779747009277,
+      "logits/rejected": -3.5263726711273193,
+      "logps/chosen": -798.533447265625,
+      "logps/rejected": -1509.49853515625,
+      "loss": 0.0685,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.5053689479827881,
+      "rewards/margins": 16.188081741333008,
+      "rewards/rejected": -15.682710647583008,
+      "step": 149
+    },
+    {
+      "epoch": 0.2546689303904924,
+      "grad_norm": 13.71898365020752,
+      "learning_rate": 6.944080280712058e-06,
+      "logits/chosen": -3.716275930404663,
+      "logits/rejected": -3.532764196395874,
+      "logps/chosen": -939.0890502929688,
+      "logps/rejected": -2572.087646484375,
+      "loss": 0.0541,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.4292657375335693,
+      "rewards/margins": 24.120561599731445,
+      "rewards/rejected": -22.691295623779297,
+      "step": 150
+    },
+    {
+      "epoch": 0.2563667232597623,
+      "grad_norm": 5.842966079711914,
+      "learning_rate": 6.943098505427058e-06,
+      "logits/chosen": -3.6546359062194824,
+      "logits/rejected": -3.6221351623535156,
+      "logps/chosen": -605.56884765625,
+      "logps/rejected": -943.8607177734375,
+      "loss": 0.1049,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.41017091274261475,
+      "rewards/margins": 10.390439987182617,
+      "rewards/rejected": -10.800610542297363,
+      "step": 151
+    },
+    {
+      "epoch": 0.25806451612903225,
+      "grad_norm": 23.814029693603516,
+      "learning_rate": 6.942108280943252e-06,
+      "logits/chosen": -3.9738707542419434,
+      "logits/rejected": -3.5914108753204346,
+      "logps/chosen": -376.2590026855469,
+      "logps/rejected": -1555.9676513671875,
+      "loss": 0.097,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": -1.7003173828125,
+      "rewards/margins": 15.424914360046387,
+      "rewards/rejected": -17.12523078918457,
+      "step": 152
+    },
+    {
+      "epoch": 0.2597623089983022,
+      "grad_norm": 18.307544708251953,
+      "learning_rate": 6.941109610518536e-06,
+      "logits/chosen": -3.872546434402466,
+      "logits/rejected": -3.487182855606079,
+      "logps/chosen": -639.40966796875,
+      "logps/rejected": -2526.517822265625,
+      "loss": 0.0721,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.2157108783721924,
+      "rewards/margins": 20.447683334350586,
+      "rewards/rejected": -21.663394927978516,
+      "step": 153
+    },
+    {
+      "epoch": 0.2614601018675722,
+      "grad_norm": 207.49671936035156,
+      "learning_rate": 6.940102497438598e-06,
+      "logits/chosen": -3.8016934394836426,
+      "logits/rejected": -3.5271739959716797,
+      "logps/chosen": -1146.2698974609375,
+      "logps/rejected": -2129.174560546875,
+      "loss": 0.9201,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": 0.18959708511829376,
+      "rewards/margins": 18.843021392822266,
+      "rewards/rejected": -18.653423309326172,
+      "step": 154
+    },
+    {
+      "epoch": 0.2631578947368421,
+      "grad_norm": 6.829913139343262,
+      "learning_rate": 6.939086945016899e-06,
+      "logits/chosen": -3.743802070617676,
+      "logits/rejected": -3.529689073562622,
+      "logps/chosen": -655.1603393554688,
+      "logps/rejected": -1918.6319580078125,
+      "loss": 0.0801,
+      "rewards/accuracies": 0.9473684430122375,
+      "rewards/chosen": -4.319693088531494,
+      "rewards/margins": 19.133678436279297,
+      "rewards/rejected": -23.453371047973633,
+      "step": 155
+    },
+    {
+      "epoch": 0.26485568760611206,
+      "grad_norm": 50.516014099121094,
+      "learning_rate": 6.9380629565946675e-06,
+      "logits/chosen": -3.3451156616210938,
+      "logits/rejected": -3.4973039627075195,
+      "logps/chosen": -900.9619750976562,
+      "logps/rejected": -1767.4691162109375,
+      "loss": 0.1874,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -3.8387162685394287,
+      "rewards/margins": 16.731220245361328,
+      "rewards/rejected": -20.569936752319336,
+      "step": 156
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 1767,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 19,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d90dff462e13a9b1d87be02fa9fa14eee18c865649dd34a288d9474609d80c9
+size 6520