Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+checkpoint-7000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-7143/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,68 @@
+---
+language: en
+license: llama3.1
+tags:
+- sentiment-analysis
+- financial-nlp
+- llama
+- peft
+- lora
+- qlora
+datasets:
+- TimKoornstra/financial-tweets-sentiment
+- TimKoornstra/synthetic-financial-tweets-sentiment
+metrics:
+- accuracy
+- f1
+pipeline_tag: text-classification
+base_model: meta-llama/Llama-3.1-8B-Instruct
+---
+
+# LLaMA 3.1-8B Fine-tuned for Financial Tweet Sentiment
+
+Fine-tuned version of `meta-llama/Llama-3.1-8B-Instruct` for financial tweet sentiment classification using QLoRA (4-bit quantization + LoRA).
+
+Inspired by [StephanAkkerman/FinTwitBERT-sentiment](https://huggingface.co/StephanAkkerman/FinTwitBERT-sentiment).
+
+## Labels
+
+| Label ID | Sentiment |
+|----------|-----------|
+| 0 | Neutral |
+| 1 | Bullish |
+| 2 | Bearish |
+
+## Training
+
+- **Base model**: `meta-llama/Llama-3.1-8B-Instruct`
+- **Method**: QLoRA (4-bit NF4 + LoRA r=16, alpha=32)
+- **Training data**: 38,091 tweets (all available data)
+- **Epochs**: 3
+- **Effective batch size**: 16
+- **Learning rate**: 0.0002
+- **Max sequence length**: 128
+- **Training time**: 9h 45m 47s
+- **Final training loss**: 1.3580
+
+## Usage
+
+```python
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+import torch
+
+tokenizer = AutoTokenizer.from_pretrained("NunoMotaRicardo/llama-3.1-8b-financial-sentiment")
+model = AutoModelForSequenceClassification.from_pretrained("NunoMotaRicardo/llama-3.1-8b-financial-sentiment")
+
+text = "$AAPL looking strong after earnings, buying more calls!"
+inputs = tokenizer(text, return_tensors="pt", truncation=True, max_length=128)
+with torch.no_grad():
+    logits = model(**inputs).logits
+pred = logits.argmax(-1).item()
+label_map = {0: "Neutral", 1: "Bullish", 2: "Bearish"}
+print(f"Sentiment: {label_map[pred]})
+```
+
+## Datasets
+
+- [TimKoornstra/financial-tweets-sentiment](https://huggingface.co/datasets/TimKoornstra/financial-tweets-sentiment)
+- [TimKoornstra/synthetic-financial-tweets-sentiment](https://huggingface.co/datasets/TimKoornstra/synthetic-financial-tweets-sentiment)

adapter_config.json

@@ -0,0 +1,49 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-3.1-8B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": [
+    "classifier",
+    "score"
+  ],
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "k_proj",
+    "q_proj",
+    "up_proj",
+    "o_proj",
+    "down_proj",
+    "v_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "SEQ_CLS",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40140e3ec79feb4e156a7f0b24d6843fcb5e1d2d1dd480e808d38509a9ce1e9e
+size 167881496

chat_template.jinja

@@ -0,0 +1,109 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- set date_string = "26 Jul 2024" %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message + builtin tools #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if builtin_tools is defined or tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{%- if builtin_tools is defined %}
+    {{- "Tools: " + builtin_tools | reject('equalto', 'code_interpreter') | join(", ") + "\n\n"}}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {%- if builtin_tools is defined and tool_call.name in builtin_tools %}
+            {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+            {{- "<|python_tag|>" + tool_call.name + ".call(" }}
+            {%- for arg_name, arg_val in tool_call.arguments | items %}
+                {{- arg_name + '="' + arg_val + '"' }}
+                {%- if not loop.last %}
+                    {{- ", " }}
+                {%- endif %}
+                {%- endfor %}
+            {{- ")" }}
+        {%- else  %}
+            {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+            {{- '{"name": "' + tool_call.name + '", ' }}
+            {{- '"parameters": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- "}" }}
+        {%- endif %}
+        {%- if builtin_tools is defined %}
+            {#- This means we're in ipython mode #}
+            {{- "<|eom_id|>" }}
+        {%- else %}
+            {{- "<|eot_id|>" }}
+        {%- endif %}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

checkpoint-7000/README.md

@@ -0,0 +1,206 @@
+---
+base_model: meta-llama/Llama-3.1-8B-Instruct
+library_name: peft
+tags:
+- base_model:adapter:meta-llama/Llama-3.1-8B-Instruct
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

checkpoint-7000/adapter_config.json

@@ -0,0 +1,49 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-3.1-8B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": [
+    "classifier",
+    "score"
+  ],
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "k_proj",
+    "q_proj",
+    "up_proj",
+    "o_proj",
+    "down_proj",
+    "v_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "SEQ_CLS",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-7000/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f30359f2592eddccc56290258aad56ea4c1ca162c13a9c0c89b02484df4760b2
+size 167881496

checkpoint-7000/chat_template.jinja

@@ -0,0 +1,109 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- set date_string = "26 Jul 2024" %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message + builtin tools #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if builtin_tools is defined or tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{%- if builtin_tools is defined %}
+    {{- "Tools: " + builtin_tools | reject('equalto', 'code_interpreter') | join(", ") + "\n\n"}}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {%- if builtin_tools is defined and tool_call.name in builtin_tools %}
+            {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+            {{- "<|python_tag|>" + tool_call.name + ".call(" }}
+            {%- for arg_name, arg_val in tool_call.arguments | items %}
+                {{- arg_name + '="' + arg_val + '"' }}
+                {%- if not loop.last %}
+                    {{- ", " }}
+                {%- endif %}
+                {%- endfor %}
+            {{- ")" }}
+        {%- else  %}
+            {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+            {{- '{"name": "' + tool_call.name + '", ' }}
+            {{- '"parameters": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- "}" }}
+        {%- endif %}
+        {%- if builtin_tools is defined %}
+            {#- This means we're in ipython mode #}
+            {{- "<|eom_id|>" }}
+        {%- else %}
+            {{- "<|eot_id|>" }}
+        {%- endif %}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

checkpoint-7000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dda377d203f02d017e71c69d626b7368197d660532afc6a4f37e44eee3394e22
+size 85755561

checkpoint-7000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ac14e73f8e5b9769e9f003c05a28363f548f3700779fe2deb19cd368d55e7c2
+size 14645

checkpoint-7000/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17d2d3c94e7430c2d6d5e80d087bdbb14ab3c964dd9866b8e23f1f0a4fb3af4e
+size 1383

checkpoint-7000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb24dc5385c380809935eb341fb39c56e56ff89b961916be3780acd42a6c07f2
+size 1465

checkpoint-7000/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbe07b35a61be0ec850f12e9379be8ca47b4ef845b20f0d146a8a2bfdb0e7e3e
+size 17210195

checkpoint-7000/tokenizer_config.json

@@ -0,0 +1,14 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "is_local": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|eot_id|>",
+  "tokenizer_class": "TokenizersBackend"
+}

checkpoint-7000/trainer_state.json

@@ -0,0 +1,1014 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.9400399033917886,
+  "eval_steps": 500,
+  "global_step": 7000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0210017851517379,
+      "grad_norm": 96.88743591308594,
+      "learning_rate": 2.0512820512820512e-05,
+      "loss": 8.165773315429687,
+      "step": 50
+    },
+    {
+      "epoch": 0.0420035703034758,
+      "grad_norm": 116.87528228759766,
+      "learning_rate": 4.382284382284382e-05,
+      "loss": 5.416732177734375,
+      "step": 100
+    },
+    {
+      "epoch": 0.0630053554552137,
+      "grad_norm": 61.64604187011719,
+      "learning_rate": 6.713286713286715e-05,
+      "loss": 4.055693664550781,
+      "step": 150
+    },
+    {
+      "epoch": 0.0840071406069516,
+      "grad_norm": 116.0433120727539,
+      "learning_rate": 9.044289044289046e-05,
+      "loss": 2.936495056152344,
+      "step": 200
+    },
+    {
+      "epoch": 0.10500892575868949,
+      "grad_norm": 151.53907775878906,
+      "learning_rate": 0.00011375291375291375,
+      "loss": 2.698266906738281,
+      "step": 250
+    },
+    {
+      "epoch": 0.1260107109104274,
+      "grad_norm": 113.19374084472656,
+      "learning_rate": 0.00013706293706293706,
+      "loss": 2.4747793579101565,
+      "step": 300
+    },
+    {
+      "epoch": 0.1470124960621653,
+      "grad_norm": 126.88410186767578,
+      "learning_rate": 0.0001603729603729604,
+      "loss": 2.5189743041992188,
+      "step": 350
+    },
+    {
+      "epoch": 0.1680142812139032,
+      "grad_norm": 59.926734924316406,
+      "learning_rate": 0.00018368298368298368,
+      "loss": 2.631827087402344,
+      "step": 400
+    },
+    {
+      "epoch": 0.18901606636564108,
+      "grad_norm": 64.80731201171875,
+      "learning_rate": 0.0001999975368672928,
+      "loss": 2.6951309204101563,
+      "step": 450
+    },
+    {
+      "epoch": 0.21001785151737898,
+      "grad_norm": 145.46435546875,
+      "learning_rate": 0.00019995375121686682,
+      "loss": 2.381460876464844,
+      "step": 500
+    },
+    {
+      "epoch": 0.23101963666911687,
+      "grad_norm": 229.77268981933594,
+      "learning_rate": 0.00019985525686985856,
+      "loss": 2.2023199462890624,
+      "step": 550
+    },
+    {
+      "epoch": 0.2520214218208548,
+      "grad_norm": 67.24784851074219,
+      "learning_rate": 0.00019970210773617416,
+      "loss": 2.550819091796875,
+      "step": 600
+    },
+    {
+      "epoch": 0.2730232069725927,
+      "grad_norm": 90.69214630126953,
+      "learning_rate": 0.00019949438764047637,
+      "loss": 2.440052032470703,
+      "step": 650
+    },
+    {
+      "epoch": 0.2940249921243306,
+      "grad_norm": 56.658851623535156,
+      "learning_rate": 0.00019923798662574877,
+      "loss": 2.2508380126953127,
+      "step": 700
+    },
+    {
+      "epoch": 0.3150267772760685,
+      "grad_norm": 24.49359130859375,
+      "learning_rate": 0.00019892258016992456,
+      "loss": 2.227517547607422,
+      "step": 750
+    },
+    {
+      "epoch": 0.3360285624278064,
+      "grad_norm": 24.36576271057129,
+      "learning_rate": 0.00019855302941878611,
+      "loss": 2.384382476806641,
+      "step": 800
+    },
+    {
+      "epoch": 0.35703034757954427,
+      "grad_norm": 59.77130889892578,
+      "learning_rate": 0.00019812953664228,
+      "loss": 2.337910614013672,
+      "step": 850
+    },
+    {
+      "epoch": 0.37803213273128217,
+      "grad_norm": 13.947885513305664,
+      "learning_rate": 0.00019765233363498648,
+      "loss": 2.007940673828125,
+      "step": 900
+    },
+    {
+      "epoch": 0.39903391788302006,
+      "grad_norm": 66.34615325927734,
+      "learning_rate": 0.00019712168158924926,
+      "loss": 2.213553924560547,
+      "step": 950
+    },
+    {
+      "epoch": 0.42003570303475796,
+      "grad_norm": 135.73300170898438,
+      "learning_rate": 0.00019653787095221418,
+      "loss": 2.4402421569824218,
+      "step": 1000
+    },
+    {
+      "epoch": 0.44103748818649585,
+      "grad_norm": 90.47286224365234,
+      "learning_rate": 0.00019590122126685597,
+      "loss": 1.987421417236328,
+      "step": 1050
+    },
+    {
+      "epoch": 0.46203927333823375,
+      "grad_norm": 27.09687614440918,
+      "learning_rate": 0.00019521208099707965,
+      "loss": 2.076619873046875,
+      "step": 1100
+    },
+    {
+      "epoch": 0.48304105848997164,
+      "grad_norm": 53.363224029541016,
+      "learning_rate": 0.00019447082733699213,
+      "loss": 2.1755889892578124,
+      "step": 1150
+    },
+    {
+      "epoch": 0.5040428436417096,
+      "grad_norm": 67.22119140625,
+      "learning_rate": 0.00019367786600444874,
+      "loss": 2.023298645019531,
+      "step": 1200
+    },
+    {
+      "epoch": 0.5250446287934475,
+      "grad_norm": 25.080530166625977,
+      "learning_rate": 0.0001928336310189877,
+      "loss": 2.353513031005859,
+      "step": 1250
+    },
+    {
+      "epoch": 0.5460464139451854,
+      "grad_norm": 65.75191497802734,
+      "learning_rate": 0.0001919385844642737,
+      "loss": 1.9453860473632814,
+      "step": 1300
+    },
+    {
+      "epoch": 0.5670481990969233,
+      "grad_norm": 100.38494873046875,
+      "learning_rate": 0.00019099321623518108,
+      "loss": 2.0582290649414063,
+      "step": 1350
+    },
+    {
+      "epoch": 0.5880499842486612,
+      "grad_norm": 30.531715393066406,
+      "learning_rate": 0.0001899980437696546,
+      "loss": 1.88087646484375,
+      "step": 1400
+    },
+    {
+      "epoch": 0.6090517694003991,
+      "grad_norm": 68.83368682861328,
+      "learning_rate": 0.000188953611765495,
+      "loss": 1.7309284973144532,
+      "step": 1450
+    },
+    {
+      "epoch": 0.630053554552137,
+      "grad_norm": 50.95656204223633,
+      "learning_rate": 0.00018786049188222403,
+      "loss": 1.8775811767578126,
+      "step": 1500
+    },
+    {
+      "epoch": 0.6510553397038749,
+      "grad_norm": 33.41248321533203,
+      "learning_rate": 0.00018671928242819226,
+      "loss": 1.945216522216797,
+      "step": 1550
+    },
+    {
+      "epoch": 0.6720571248556128,
+      "grad_norm": 44.737709045410156,
+      "learning_rate": 0.00018553060803310083,
+      "loss": 2.106058807373047,
+      "step": 1600
+    },
+    {
+      "epoch": 0.6930589100073506,
+      "grad_norm": 35.77622985839844,
+      "learning_rate": 0.0001842951193061168,
+      "loss": 1.7817611694335938,
+      "step": 1650
+    },
+    {
+      "epoch": 0.7140606951590885,
+      "grad_norm": 29.49221420288086,
+      "learning_rate": 0.0001830134924797684,
+      "loss": 1.8394444274902344,
+      "step": 1700
+    },
+    {
+      "epoch": 0.7350624803108264,
+      "grad_norm": 74.85762786865234,
+      "learning_rate": 0.00018168642903981607,
+      "loss": 1.8008517456054687,
+      "step": 1750
+    },
+    {
+      "epoch": 0.7560642654625643,
+      "grad_norm": 54.40049743652344,
+      "learning_rate": 0.0001803146553413009,
+      "loss": 1.8719265747070313,
+      "step": 1800
+    },
+    {
+      "epoch": 0.7770660506143022,
+      "grad_norm": 68.5858154296875,
+      "learning_rate": 0.00017889892221098163,
+      "loss": 1.93374267578125,
+      "step": 1850
+    },
+    {
+      "epoch": 0.7980678357660401,
+      "grad_norm": 43.123600006103516,
+      "learning_rate": 0.00017744000453637711,
+      "loss": 1.931842041015625,
+      "step": 1900
+    },
+    {
+      "epoch": 0.819069620917778,
+      "grad_norm": 42.68077087402344,
+      "learning_rate": 0.00017593870084163892,
+      "loss": 1.932684326171875,
+      "step": 1950
+    },
+    {
+      "epoch": 0.8400714060695159,
+      "grad_norm": 32.452735900878906,
+      "learning_rate": 0.00017439583285048748,
+      "loss": 2.020773468017578,
+      "step": 2000
+    },
+    {
+      "epoch": 0.8610731912212538,
+      "grad_norm": 23.90448760986328,
+      "learning_rate": 0.00017281224503644944,
+      "loss": 1.7309165954589845,
+      "step": 2050
+    },
+    {
+      "epoch": 0.8820749763729917,
+      "grad_norm": 122.4535140991211,
+      "learning_rate": 0.00017118880416064353,
+      "loss": 1.9589013671875,
+      "step": 2100
+    },
+    {
+      "epoch": 0.9030767615247296,
+      "grad_norm": 27.976648330688477,
+      "learning_rate": 0.00016952639879736755,
+      "loss": 1.9594146728515625,
+      "step": 2150
+    },
+    {
+      "epoch": 0.9240785466764675,
+      "grad_norm": 32.92108917236328,
+      "learning_rate": 0.00016782593884774586,
+      "loss": 1.8159463500976563,
+      "step": 2200
+    },
+    {
+      "epoch": 0.9450803318282054,
+      "grad_norm": 110.42147064208984,
+      "learning_rate": 0.00016608835504170396,
+      "loss": 1.8676722717285157,
+      "step": 2250
+    },
+    {
+      "epoch": 0.9660821169799433,
+      "grad_norm": 83.26129913330078,
+      "learning_rate": 0.00016431459842854303,
+      "loss": 1.955204620361328,
+      "step": 2300
+    },
+    {
+      "epoch": 0.9870839021316812,
+      "grad_norm": 41.638790130615234,
+      "learning_rate": 0.00016250563985639204,
+      "loss": 1.8278160095214844,
+      "step": 2350
+    },
+    {
+      "epoch": 1.0079806783576604,
+      "grad_norm": 11.446990013122559,
+      "learning_rate": 0.00016066246944082394,
+      "loss": 1.7912734985351562,
+      "step": 2400
+    },
+    {
+      "epoch": 1.0289824635093983,
+      "grad_norm": 5.880982398986816,
+      "learning_rate": 0.00015878609602292562,
+      "loss": 1.458508758544922,
+      "step": 2450
+    },
+    {
+      "epoch": 1.0499842486611362,
+      "grad_norm": 99.73579406738281,
+      "learning_rate": 0.00015687754661711866,
+      "loss": 1.5889370727539063,
+      "step": 2500
+    },
+    {
+      "epoch": 1.070986033812874,
+      "grad_norm": 30.614892959594727,
+      "learning_rate": 0.0001549378658490333,
+      "loss": 1.396112823486328,
+      "step": 2550
+    },
+    {
+      "epoch": 1.091987818964612,
+      "grad_norm": 42.81917953491211,
+      "learning_rate": 0.00015296811538374278,
+      "loss": 1.2478562927246093,
+      "step": 2600
+    },
+    {
+      "epoch": 1.1129896041163498,
+      "grad_norm": 89.29583740234375,
+      "learning_rate": 0.00015096937334467151,
+      "loss": 1.6881478881835938,
+      "step": 2650
+    },
+    {
+      "epoch": 1.1339913892680877,
+      "grad_norm": 45.66031265258789,
+      "learning_rate": 0.000148942733723495,
+      "loss": 1.3388128662109375,
+      "step": 2700
+    },
+    {
+      "epoch": 1.1549931744198256,
+      "grad_norm": 19.098651885986328,
+      "learning_rate": 0.00014688930578135416,
+      "loss": 1.4715719604492188,
+      "step": 2750
+    },
+    {
+      "epoch": 1.1759949595715635,
+      "grad_norm": 40.75487518310547,
+      "learning_rate": 0.0001448102134417123,
+      "loss": 1.3008560180664062,
+      "step": 2800
+    },
+    {
+      "epoch": 1.1969967447233014,
+      "grad_norm": NaN,
+      "learning_rate": 0.00014270659467518682,
+      "loss": 1.458815155029297,
+      "step": 2850
+    },
+    {
+      "epoch": 1.2179985298750393,
+      "grad_norm": 59.06400680541992,
+      "learning_rate": 0.00014062236231847836,
+      "loss": 1.4431008911132812,
+      "step": 2900
+    },
+    {
+      "epoch": 1.2390003150267772,
+      "grad_norm": 55.12430953979492,
+      "learning_rate": 0.00013847359040513967,
+      "loss": 1.2680451965332031,
+      "step": 2950
+    },
+    {
+      "epoch": 1.260002100178515,
+      "grad_norm": 18.353561401367188,
+      "learning_rate": 0.0001363037603528289,
+      "loss": 1.3490068054199218,
+      "step": 3000
+    },
+    {
+      "epoch": 1.281003885330253,
+      "grad_norm": 46.3448371887207,
+      "learning_rate": 0.00013411405979655243,
+      "loss": 1.31285400390625,
+      "step": 3050
+    },
+    {
+      "epoch": 1.302005670481991,
+      "grad_norm": 28.819128036499023,
+      "learning_rate": 0.0001319056872472404,
+      "loss": 1.2427201080322265,
+      "step": 3100
+    },
+    {
+      "epoch": 1.3230074556337288,
+      "grad_norm": 41.59355545043945,
+      "learning_rate": 0.00012967985143575356,
+      "loss": 1.488096923828125,
+      "step": 3150
+    },
+    {
+      "epoch": 1.3440092407854667,
+      "grad_norm": 21.26289939880371,
+      "learning_rate": 0.00012743777065129635,
+      "loss": 1.3544757080078125,
+      "step": 3200
+    },
+    {
+      "epoch": 1.3650110259372046,
+      "grad_norm": 42.516868591308594,
+      "learning_rate": 0.0001251806720745984,
+      "loss": 1.3172706604003905,
+      "step": 3250
+    },
+    {
+      "epoch": 1.3860128110889427,
+      "grad_norm": 51.99966812133789,
+      "learning_rate": 0.00012290979110622937,
+      "loss": 1.340508270263672,
+      "step": 3300
+    },
+    {
+      "epoch": 1.4070145962406806,
+      "grad_norm": 40.60024642944336,
+      "learning_rate": 0.00012062637069041456,
+      "loss": 1.2897941589355468,
+      "step": 3350
+    },
+    {
+      "epoch": 1.4280163813924185,
+      "grad_norm": 31.378459930419922,
+      "learning_rate": 0.0001183316606347217,
+      "loss": 1.5410649108886718,
+      "step": 3400
+    },
+    {
+      "epoch": 1.4490181665441564,
+      "grad_norm": 26.510839462280273,
+      "learning_rate": 0.00011602691692599127,
+      "loss": 1.2167890167236328,
+      "step": 3450
+    },
+    {
+      "epoch": 1.4700199516958943,
+      "grad_norm": 23.495378494262695,
+      "learning_rate": 0.00011371340104288442,
+      "loss": 1.42876708984375,
+      "step": 3500
+    },
+    {
+      "epoch": 1.4910217368476322,
+      "grad_norm": 28.301671981811523,
+      "learning_rate": 0.00011139237926542509,
+      "loss": 1.063271255493164,
+      "step": 3550
+    },
+    {
+      "epoch": 1.51202352199937,
+      "grad_norm": 21.21889877319336,
+      "learning_rate": 0.0001090651219819143,
+      "loss": 1.2190794372558593,
+      "step": 3600
+    },
+    {
+      "epoch": 1.533025307151108,
+      "grad_norm": 15.157039642333984,
+      "learning_rate": 0.00010673290299359551,
+      "loss": 1.2155840301513672,
+      "step": 3650
+    },
+    {
+      "epoch": 1.5540270923028459,
+      "grad_norm": 12.83481502532959,
+      "learning_rate": 0.00010439699881745222,
+      "loss": 1.2669831085205079,
+      "step": 3700
+    },
+    {
+      "epoch": 1.5750288774545838,
+      "grad_norm": 40.47445297241211,
+      "learning_rate": 0.00010205868798751908,
+      "loss": 1.189565658569336,
+      "step": 3750
+    },
+    {
+      "epoch": 1.5960306626063216,
+      "grad_norm": 47.97238540649414,
+      "learning_rate": 9.971925035508882e-05,
+      "loss": 1.3227517700195313,
+      "step": 3800
+    },
+    {
+      "epoch": 1.6170324477580595,
+      "grad_norm": 11.237785339355469,
+      "learning_rate": 9.737996638819842e-05,
+      "loss": 1.1511013793945313,
+      "step": 3850
+    },
+    {
+      "epoch": 1.6380342329097974,
+      "grad_norm": 27.457271575927734,
+      "learning_rate": 9.504211647077748e-05,
+      "loss": 1.1926652526855468,
+      "step": 3900
+    },
+    {
+      "epoch": 1.6590360180615353,
+      "grad_norm": 15.741539001464844,
+      "learning_rate": 9.270698020184285e-05,
+      "loss": 1.0482740020751953,
+      "step": 3950
+    },
+    {
+      "epoch": 1.6800378032132732,
+      "grad_norm": 7.643248558044434,
+      "learning_rate": 9.037583569512271e-05,
+      "loss": 1.2052240753173828,
+      "step": 4000
+    },
+    {
+      "epoch": 1.7010395883650111,
+      "grad_norm": 20.112401962280273,
+      "learning_rate": 8.804995887949373e-05,
+      "loss": 1.0017765808105468,
+      "step": 4050
+    },
+    {
+      "epoch": 1.722041373516749,
+      "grad_norm": 22.75946617126465,
+      "learning_rate": 8.573062280061399e-05,
+      "loss": 1.348891143798828,
+      "step": 4100
+    },
+    {
+      "epoch": 1.743043158668487,
+      "grad_norm": 47.762718200683594,
+      "learning_rate": 8.341909692413401e-05,
+      "loss": 1.1363021850585937,
+      "step": 4150
+    },
+    {
+      "epoch": 1.7640449438202248,
+      "grad_norm": 28.285919189453125,
+      "learning_rate": 8.111664644086733e-05,
+      "loss": 1.1882960510253906,
+      "step": 4200
+    },
+    {
+      "epoch": 1.7850467289719627,
+      "grad_norm": 6.3662214279174805,
+      "learning_rate": 7.882453157430086e-05,
+      "loss": 1.21821044921875,
+      "step": 4250
+    },
+    {
+      "epoch": 1.8060485141237006,
+      "grad_norm": 11.175128936767578,
+      "learning_rate": 7.654400689082387e-05,
+      "loss": 1.1443672180175781,
+      "step": 4300
+    },
+    {
+      "epoch": 1.8270502992754385,
+      "grad_norm": 7.028075218200684,
+      "learning_rate": 7.427632061305376e-05,
+      "loss": 1.1513980865478515,
+      "step": 4350
+    },
+    {
+      "epoch": 1.8480520844271764,
+      "grad_norm": 3.8552818298339844,
+      "learning_rate": 7.202271393663354e-05,
+      "loss": 1.1591477966308594,
+      "step": 4400
+    },
+    {
+      "epoch": 1.8690538695789143,
+      "grad_norm": 27.834590911865234,
+      "learning_rate": 6.978442035087579e-05,
+      "loss": 1.0166816711425781,
+      "step": 4450
+    },
+    {
+      "epoch": 1.8900556547306522,
+      "grad_norm": 34.69282913208008,
+      "learning_rate": 6.756266496362442e-05,
+      "loss": 1.1785438537597657,
+      "step": 4500
+    },
+    {
+      "epoch": 1.91105743988239,
+      "grad_norm": 34.50713348388672,
+      "learning_rate": 6.535866383070405e-05,
+      "loss": 0.9877433013916016,
+      "step": 4550
+    },
+    {
+      "epoch": 1.932059225034128,
+      "grad_norm": 28.763914108276367,
+      "learning_rate": 6.317362329032388e-05,
+      "loss": 0.9768619537353516,
+      "step": 4600
+    },
+    {
+      "epoch": 1.9530610101858659,
+      "grad_norm": 44.40330505371094,
+      "learning_rate": 6.1008739302800486e-05,
+      "loss": 1.2748094940185546,
+      "step": 4650
+    },
+    {
+      "epoch": 1.9740627953376038,
+      "grad_norm": 45.88819122314453,
+      "learning_rate": 5.886519679596063e-05,
+      "loss": 0.8840785980224609,
+      "step": 4700
+    },
+    {
+      "epoch": 1.9950645804893417,
+      "grad_norm": 26.39780044555664,
+      "learning_rate": 5.674416901658288e-05,
+      "loss": 1.1054458618164062,
+      "step": 4750
+    },
+    {
+      "epoch": 2.0159613567153207,
+      "grad_norm": 28.105772018432617,
+      "learning_rate": 5.464681688823246e-05,
+      "loss": 0.43650150299072266,
+      "step": 4800
+    },
+    {
+      "epoch": 2.0369631418670586,
+      "grad_norm": 0.8811307549476624,
+      "learning_rate": 5.257428837584142e-05,
+      "loss": 0.4781087112426758,
+      "step": 4850
+    },
+    {
+      "epoch": 2.0579649270187965,
+      "grad_norm": 29.04726791381836,
+      "learning_rate": 5.052771785738109e-05,
+      "loss": 0.538114242553711,
+      "step": 4900
+    },
+    {
+      "epoch": 2.0789667121705344,
+      "grad_norm": 2.272526741027832,
+      "learning_rate": 4.8548342809611756e-05,
+      "loss": 0.5318920516967773,
+      "step": 4950
+    },
+    {
+      "epoch": 2.0999684973222723,
+      "grad_norm": 35.368194580078125,
+      "learning_rate": 4.655645956450846e-05,
+      "loss": 0.48801342010498044,
+      "step": 5000
+    },
+    {
+      "epoch": 2.12097028247401,
+      "grad_norm": 0.03809299319982529,
+      "learning_rate": 4.4593828112383176e-05,
+      "loss": 0.4779471206665039,
+      "step": 5050
+    },
+    {
+      "epoch": 2.141972067625748,
+      "grad_norm": 8.704736709594727,
+      "learning_rate": 4.269986472510045e-05,
+      "loss": 0.6272232437133789,
+      "step": 5100
+    },
+    {
+      "epoch": 2.162973852777486,
+      "grad_norm": 2.0720932483673096,
+      "learning_rate": 4.0798305013432315e-05,
+      "loss": 0.43107017517089846,
+      "step": 5150
+    },
+    {
+      "epoch": 2.183975637929224,
+      "grad_norm": 8.66932201385498,
+      "learning_rate": 3.8929148763652736e-05,
+      "loss": 0.5629381561279296,
+      "step": 5200
+    },
+    {
+      "epoch": 2.204977423080962,
+      "grad_norm": 36.49018096923828,
+      "learning_rate": 3.7093419039937825e-05,
+      "loss": 0.6745287322998047,
+      "step": 5250
+    },
+    {
+      "epoch": 2.2259792082326997,
+      "grad_norm": 44.782039642333984,
+      "learning_rate": 3.529212061078543e-05,
+      "loss": 0.44454833984375,
+      "step": 5300
+    },
+    {
+      "epoch": 2.2469809933844376,
+      "grad_norm": 38.312313079833984,
+      "learning_rate": 3.352623939906491e-05,
+      "loss": 0.4252283477783203,
+      "step": 5350
+    },
+    {
+      "epoch": 2.2679827785361755,
+      "grad_norm": 48.045509338378906,
+      "learning_rate": 3.179674194238237e-05,
+      "loss": 0.6742375183105469,
+      "step": 5400
+    },
+    {
+      "epoch": 2.2889845636879134,
+      "grad_norm": 63.853843688964844,
+      "learning_rate": 3.010457486405549e-05,
+      "loss": 0.4491603851318359,
+      "step": 5450
+    },
+    {
+      "epoch": 2.3099863488396513,
+      "grad_norm": 1.4884780645370483,
+      "learning_rate": 2.84506643549889e-05,
+      "loss": 0.5363286972045899,
+      "step": 5500
+    },
+    {
+      "epoch": 2.330988133991389,
+      "grad_norm": 0.02728688158094883,
+      "learning_rate": 2.6835915666732548e-05,
+      "loss": 0.34341278076171877,
+      "step": 5550
+    },
+    {
+      "epoch": 2.351989919143127,
+      "grad_norm": 46.706119537353516,
+      "learning_rate": 2.5261212616001327e-05,
+      "loss": 0.6819667816162109,
+      "step": 5600
+    },
+    {
+      "epoch": 2.372991704294865,
+      "grad_norm": 2.9122889041900635,
+      "learning_rate": 2.372741710092674e-05,
+      "loss": 0.3454324722290039,
+      "step": 5650
+    },
+    {
+      "epoch": 2.393993489446603,
+      "grad_norm": 2.8348746299743652,
+      "learning_rate": 2.2235368629305418e-05,
+      "loss": 0.37347347259521485,
+      "step": 5700
+    },
+    {
+      "epoch": 2.4149952745983407,
+      "grad_norm": 2.396700859069824,
+      "learning_rate": 2.0785883859102994e-05,
+      "loss": 0.478424072265625,
+      "step": 5750
+    },
+    {
+      "epoch": 2.4359970597500786,
+      "grad_norm": 0.3374341130256653,
+      "learning_rate": 1.9379756151464236e-05,
+      "loss": 0.3879497528076172,
+      "step": 5800
+    },
+    {
+      "epoch": 2.4569988449018165,
+      "grad_norm": 0.07735343277454376,
+      "learning_rate": 1.801775513647487e-05,
+      "loss": 0.3089980697631836,
+      "step": 5850
+    },
+    {
+      "epoch": 2.4780006300535544,
+      "grad_norm": 45.71025848388672,
+      "learning_rate": 1.6700626291912004e-05,
+      "loss": 0.3591925048828125,
+      "step": 5900
+    },
+    {
+      "epoch": 2.4990024152052923,
+      "grad_norm": 86.34896850585938,
+      "learning_rate": 1.5429090535214353e-05,
+      "loss": 0.4602979278564453,
+      "step": 5950
+    },
+    {
+      "epoch": 2.52000420035703,
+      "grad_norm": 125.41795349121094,
+      "learning_rate": 1.420384382889518e-05,
+      "loss": 0.4123740005493164,
+      "step": 6000
+    },
+    {
+      "epoch": 2.541005985508768,
+      "grad_norm": 0.06989021599292755,
+      "learning_rate": 1.302555679961408e-05,
+      "loss": 0.5613171768188476,
+      "step": 6050
+    },
+    {
+      "epoch": 2.562007770660506,
+      "grad_norm": 50.419254302978516,
+      "learning_rate": 1.1894874371116182e-05,
+      "loss": 0.44726715087890623,
+      "step": 6100
+    },
+    {
+      "epoch": 2.583009555812244,
+      "grad_norm": 29.774009704589844,
+      "learning_rate": 1.081241541123944e-05,
+      "loss": 0.36859161376953126,
+      "step": 6150
+    },
+    {
+      "epoch": 2.604011340963982,
+      "grad_norm": 154.20655822753906,
+      "learning_rate": 9.778772393183467e-06,
+      "loss": 0.5629509735107422,
+      "step": 6200
+    },
+    {
+      "epoch": 2.6250131261157197,
+      "grad_norm": 0.09836237877607346,
+      "learning_rate": 8.794511071225275e-06,
+      "loss": 0.371593132019043,
+      "step": 6250
+    },
+    {
+      "epoch": 2.6460149112674576,
+      "grad_norm": 56.41450500488281,
+      "learning_rate": 7.860170171059067e-06,
+      "loss": 0.44127925872802737,
+      "step": 6300
+    },
+    {
+      "epoch": 2.6670166964191955,
+      "grad_norm": 30.18195152282715,
+      "learning_rate": 6.976261094930347e-06,
+      "loss": 0.5457481384277344,
+      "step": 6350
+    },
+    {
+      "epoch": 2.6880184815709334,
+      "grad_norm": 1.9519455432891846,
+      "learning_rate": 6.1432676417246925e-06,
+      "loss": 0.46905494689941407,
+      "step": 6400
+    },
+    {
+      "epoch": 2.7090202667226713,
+      "grad_norm": 0.3305146396160126,
+      "learning_rate": 5.3616457421654e-06,
+      "loss": 0.18893402099609374,
+      "step": 6450
+    },
+    {
+      "epoch": 2.730022051874409,
+      "grad_norm": 0.018915779888629913,
+      "learning_rate": 4.63182320926423e-06,
+      "loss": 0.5251737594604492,
+      "step": 6500
+    },
+    {
+      "epoch": 2.751023837026147,
+      "grad_norm": 2.023514986038208,
+      "learning_rate": 3.9541995041621925e-06,
+      "loss": 0.3400577163696289,
+      "step": 6550
+    },
+    {
+      "epoch": 2.7720256221778854,
+      "grad_norm": 27.531789779663086,
+      "learning_rate": 3.32914551748853e-06,
+      "loss": 0.4619370269775391,
+      "step": 6600
+    },
+    {
+      "epoch": 2.793027407329623,
+      "grad_norm": 23.563621520996094,
+      "learning_rate": 2.757003366357236e-06,
+      "loss": 0.3556163787841797,
+      "step": 6650
+    },
+    {
+      "epoch": 2.814029192481361,
+      "grad_norm": 0.1751919537782669,
+      "learning_rate": 2.2380862071128327e-06,
+      "loss": 0.43869766235351565,
+      "step": 6700
+    },
+    {
+      "epoch": 2.8350309776330986,
+      "grad_norm": 74.48571014404297,
+      "learning_rate": 1.77267806392718e-06,
+      "loss": 0.43838497161865236,
+      "step": 6750
+    },
+    {
+      "epoch": 2.856032762784837,
+      "grad_norm": 0.02355477400124073,
+      "learning_rate": 1.3610336733417605e-06,
+      "loss": 0.27370246887207034,
+      "step": 6800
+    },
+    {
+      "epoch": 2.8770345479365744,
+      "grad_norm": 79.37924194335938,
+      "learning_rate": 1.0033783448401112e-06,
+      "loss": 0.521487808227539,
+      "step": 6850
+    },
+    {
+      "epoch": 2.8980363330883128,
+      "grad_norm": 0.01911379024386406,
+      "learning_rate": 6.999078375269252e-07,
+      "loss": 0.424820671081543,
+      "step": 6900
+    },
+    {
+      "epoch": 2.91903811824005,
+      "grad_norm": 63.20115661621094,
+      "learning_rate": 4.5078825298122683e-07,
+      "loss": 0.48077041625976563,
+      "step": 6950
+    },
+    {
+      "epoch": 2.9400399033917886,
+      "grad_norm": 19.453731536865234,
+      "learning_rate": 2.5615594434229564e-07,
+      "loss": 0.4773090362548828,
+      "step": 7000
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 7143,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 6.039122105755238e+17,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-7000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3dcfe9cb7f80cca256d11b427979266d513b9e02e5d47123e2e934069b9ff0d4
+size 5265

checkpoint-7143/README.md

@@ -0,0 +1,206 @@
+---
+base_model: meta-llama/Llama-3.1-8B-Instruct
+library_name: peft
+tags:
+- base_model:adapter:meta-llama/Llama-3.1-8B-Instruct
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

checkpoint-7143/adapter_config.json

@@ -0,0 +1,49 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-3.1-8B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": [
+    "classifier",
+    "score"
+  ],
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "k_proj",
+    "q_proj",
+    "up_proj",
+    "o_proj",
+    "down_proj",
+    "v_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "SEQ_CLS",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-7143/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40140e3ec79feb4e156a7f0b24d6843fcb5e1d2d1dd480e808d38509a9ce1e9e
+size 167881496

checkpoint-7143/chat_template.jinja

@@ -0,0 +1,109 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- set date_string = "26 Jul 2024" %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message + builtin tools #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if builtin_tools is defined or tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{%- if builtin_tools is defined %}
+    {{- "Tools: " + builtin_tools | reject('equalto', 'code_interpreter') | join(", ") + "\n\n"}}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {%- if builtin_tools is defined and tool_call.name in builtin_tools %}
+            {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+            {{- "<|python_tag|>" + tool_call.name + ".call(" }}
+            {%- for arg_name, arg_val in tool_call.arguments | items %}
+                {{- arg_name + '="' + arg_val + '"' }}
+                {%- if not loop.last %}
+                    {{- ", " }}
+                {%- endif %}
+                {%- endfor %}
+            {{- ")" }}
+        {%- else  %}
+            {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+            {{- '{"name": "' + tool_call.name + '", ' }}
+            {{- '"parameters": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- "}" }}
+        {%- endif %}
+        {%- if builtin_tools is defined %}
+            {#- This means we're in ipython mode #}
+            {{- "<|eom_id|>" }}
+        {%- else %}
+            {{- "<|eot_id|>" }}
+        {%- endif %}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

checkpoint-7143/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc6afe84c0e9f82c581b2253a3a4a0878c148db870c6f230adb4a4ec6e79be3e
+size 85755561

checkpoint-7143/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c99f9105dc37c265a596c031fe7a6820494b607d84a3cde5e82fe07b105dae24
+size 14645

checkpoint-7143/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8850932d9acfacf0feb268963dc6684a46c50f514e0528a879fa32a2cb085c31
+size 1383

checkpoint-7143/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8904945884075cc1b1c5d7c94d4927320f40ce65ff08f5c590f2906a4d25f20
+size 1465

checkpoint-7143/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbe07b35a61be0ec850f12e9379be8ca47b4ef845b20f0d146a8a2bfdb0e7e3e
+size 17210195

checkpoint-7143/tokenizer_config.json

@@ -0,0 +1,14 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "is_local": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|eot_id|>",
+  "tokenizer_class": "TokenizersBackend"
+}

checkpoint-7143/trainer_state.json

@@ -0,0 +1,1028 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 7143,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0210017851517379,
+      "grad_norm": 96.88743591308594,
+      "learning_rate": 2.0512820512820512e-05,
+      "loss": 8.165773315429687,
+      "step": 50
+    },
+    {
+      "epoch": 0.0420035703034758,
+      "grad_norm": 116.87528228759766,
+      "learning_rate": 4.382284382284382e-05,
+      "loss": 5.416732177734375,
+      "step": 100
+    },
+    {
+      "epoch": 0.0630053554552137,
+      "grad_norm": 61.64604187011719,
+      "learning_rate": 6.713286713286715e-05,
+      "loss": 4.055693664550781,
+      "step": 150
+    },
+    {
+      "epoch": 0.0840071406069516,
+      "grad_norm": 116.0433120727539,
+      "learning_rate": 9.044289044289046e-05,
+      "loss": 2.936495056152344,
+      "step": 200
+    },
+    {
+      "epoch": 0.10500892575868949,
+      "grad_norm": 151.53907775878906,
+      "learning_rate": 0.00011375291375291375,
+      "loss": 2.698266906738281,
+      "step": 250
+    },
+    {
+      "epoch": 0.1260107109104274,
+      "grad_norm": 113.19374084472656,
+      "learning_rate": 0.00013706293706293706,
+      "loss": 2.4747793579101565,
+      "step": 300
+    },
+    {
+      "epoch": 0.1470124960621653,
+      "grad_norm": 126.88410186767578,
+      "learning_rate": 0.0001603729603729604,
+      "loss": 2.5189743041992188,
+      "step": 350
+    },
+    {
+      "epoch": 0.1680142812139032,
+      "grad_norm": 59.926734924316406,
+      "learning_rate": 0.00018368298368298368,
+      "loss": 2.631827087402344,
+      "step": 400
+    },
+    {
+      "epoch": 0.18901606636564108,
+      "grad_norm": 64.80731201171875,
+      "learning_rate": 0.0001999975368672928,
+      "loss": 2.6951309204101563,
+      "step": 450
+    },
+    {
+      "epoch": 0.21001785151737898,
+      "grad_norm": 145.46435546875,
+      "learning_rate": 0.00019995375121686682,
+      "loss": 2.381460876464844,
+      "step": 500
+    },
+    {
+      "epoch": 0.23101963666911687,
+      "grad_norm": 229.77268981933594,
+      "learning_rate": 0.00019985525686985856,
+      "loss": 2.2023199462890624,
+      "step": 550
+    },
+    {
+      "epoch": 0.2520214218208548,
+      "grad_norm": 67.24784851074219,
+      "learning_rate": 0.00019970210773617416,
+      "loss": 2.550819091796875,
+      "step": 600
+    },
+    {
+      "epoch": 0.2730232069725927,
+      "grad_norm": 90.69214630126953,
+      "learning_rate": 0.00019949438764047637,
+      "loss": 2.440052032470703,
+      "step": 650
+    },
+    {
+      "epoch": 0.2940249921243306,
+      "grad_norm": 56.658851623535156,
+      "learning_rate": 0.00019923798662574877,
+      "loss": 2.2508380126953127,
+      "step": 700
+    },
+    {
+      "epoch": 0.3150267772760685,
+      "grad_norm": 24.49359130859375,
+      "learning_rate": 0.00019892258016992456,
+      "loss": 2.227517547607422,
+      "step": 750
+    },
+    {
+      "epoch": 0.3360285624278064,
+      "grad_norm": 24.36576271057129,
+      "learning_rate": 0.00019855302941878611,
+      "loss": 2.384382476806641,
+      "step": 800
+    },
+    {
+      "epoch": 0.35703034757954427,
+      "grad_norm": 59.77130889892578,
+      "learning_rate": 0.00019812953664228,
+      "loss": 2.337910614013672,
+      "step": 850
+    },
+    {
+      "epoch": 0.37803213273128217,
+      "grad_norm": 13.947885513305664,
+      "learning_rate": 0.00019765233363498648,
+      "loss": 2.007940673828125,
+      "step": 900
+    },
+    {
+      "epoch": 0.39903391788302006,
+      "grad_norm": 66.34615325927734,
+      "learning_rate": 0.00019712168158924926,
+      "loss": 2.213553924560547,
+      "step": 950
+    },
+    {
+      "epoch": 0.42003570303475796,
+      "grad_norm": 135.73300170898438,
+      "learning_rate": 0.00019653787095221418,
+      "loss": 2.4402421569824218,
+      "step": 1000
+    },
+    {
+      "epoch": 0.44103748818649585,
+      "grad_norm": 90.47286224365234,
+      "learning_rate": 0.00019590122126685597,
+      "loss": 1.987421417236328,
+      "step": 1050
+    },
+    {
+      "epoch": 0.46203927333823375,
+      "grad_norm": 27.09687614440918,
+      "learning_rate": 0.00019521208099707965,
+      "loss": 2.076619873046875,
+      "step": 1100
+    },
+    {
+      "epoch": 0.48304105848997164,
+      "grad_norm": 53.363224029541016,
+      "learning_rate": 0.00019447082733699213,
+      "loss": 2.1755889892578124,
+      "step": 1150
+    },
+    {
+      "epoch": 0.5040428436417096,
+      "grad_norm": 67.22119140625,
+      "learning_rate": 0.00019367786600444874,
+      "loss": 2.023298645019531,
+      "step": 1200
+    },
+    {
+      "epoch": 0.5250446287934475,
+      "grad_norm": 25.080530166625977,
+      "learning_rate": 0.0001928336310189877,
+      "loss": 2.353513031005859,
+      "step": 1250
+    },
+    {
+      "epoch": 0.5460464139451854,
+      "grad_norm": 65.75191497802734,
+      "learning_rate": 0.0001919385844642737,
+      "loss": 1.9453860473632814,
+      "step": 1300
+    },
+    {
+      "epoch": 0.5670481990969233,
+      "grad_norm": 100.38494873046875,
+      "learning_rate": 0.00019099321623518108,
+      "loss": 2.0582290649414063,
+      "step": 1350
+    },
+    {
+      "epoch": 0.5880499842486612,
+      "grad_norm": 30.531715393066406,
+      "learning_rate": 0.0001899980437696546,
+      "loss": 1.88087646484375,
+      "step": 1400
+    },
+    {
+      "epoch": 0.6090517694003991,
+      "grad_norm": 68.83368682861328,
+      "learning_rate": 0.000188953611765495,
+      "loss": 1.7309284973144532,
+      "step": 1450
+    },
+    {
+      "epoch": 0.630053554552137,
+      "grad_norm": 50.95656204223633,
+      "learning_rate": 0.00018786049188222403,
+      "loss": 1.8775811767578126,
+      "step": 1500
+    },
+    {
+      "epoch": 0.6510553397038749,
+      "grad_norm": 33.41248321533203,
+      "learning_rate": 0.00018671928242819226,
+      "loss": 1.945216522216797,
+      "step": 1550
+    },
+    {
+      "epoch": 0.6720571248556128,
+      "grad_norm": 44.737709045410156,
+      "learning_rate": 0.00018553060803310083,
+      "loss": 2.106058807373047,
+      "step": 1600
+    },
+    {
+      "epoch": 0.6930589100073506,
+      "grad_norm": 35.77622985839844,
+      "learning_rate": 0.0001842951193061168,
+      "loss": 1.7817611694335938,
+      "step": 1650
+    },
+    {
+      "epoch": 0.7140606951590885,
+      "grad_norm": 29.49221420288086,
+      "learning_rate": 0.0001830134924797684,
+      "loss": 1.8394444274902344,
+      "step": 1700
+    },
+    {
+      "epoch": 0.7350624803108264,
+      "grad_norm": 74.85762786865234,
+      "learning_rate": 0.00018168642903981607,
+      "loss": 1.8008517456054687,
+      "step": 1750
+    },
+    {
+      "epoch": 0.7560642654625643,
+      "grad_norm": 54.40049743652344,
+      "learning_rate": 0.0001803146553413009,
+      "loss": 1.8719265747070313,
+      "step": 1800
+    },
+    {
+      "epoch": 0.7770660506143022,
+      "grad_norm": 68.5858154296875,
+      "learning_rate": 0.00017889892221098163,
+      "loss": 1.93374267578125,
+      "step": 1850
+    },
+    {
+      "epoch": 0.7980678357660401,
+      "grad_norm": 43.123600006103516,
+      "learning_rate": 0.00017744000453637711,
+      "loss": 1.931842041015625,
+      "step": 1900
+    },
+    {
+      "epoch": 0.819069620917778,
+      "grad_norm": 42.68077087402344,
+      "learning_rate": 0.00017593870084163892,
+      "loss": 1.932684326171875,
+      "step": 1950
+    },
+    {
+      "epoch": 0.8400714060695159,
+      "grad_norm": 32.452735900878906,
+      "learning_rate": 0.00017439583285048748,
+      "loss": 2.020773468017578,
+      "step": 2000
+    },
+    {
+      "epoch": 0.8610731912212538,
+      "grad_norm": 23.90448760986328,
+      "learning_rate": 0.00017281224503644944,
+      "loss": 1.7309165954589845,
+      "step": 2050
+    },
+    {
+      "epoch": 0.8820749763729917,
+      "grad_norm": 122.4535140991211,
+      "learning_rate": 0.00017118880416064353,
+      "loss": 1.9589013671875,
+      "step": 2100
+    },
+    {
+      "epoch": 0.9030767615247296,
+      "grad_norm": 27.976648330688477,
+      "learning_rate": 0.00016952639879736755,
+      "loss": 1.9594146728515625,
+      "step": 2150
+    },
+    {
+      "epoch": 0.9240785466764675,
+      "grad_norm": 32.92108917236328,
+      "learning_rate": 0.00016782593884774586,
+      "loss": 1.8159463500976563,
+      "step": 2200
+    },
+    {
+      "epoch": 0.9450803318282054,
+      "grad_norm": 110.42147064208984,
+      "learning_rate": 0.00016608835504170396,
+      "loss": 1.8676722717285157,
+      "step": 2250
+    },
+    {
+      "epoch": 0.9660821169799433,
+      "grad_norm": 83.26129913330078,
+      "learning_rate": 0.00016431459842854303,
+      "loss": 1.955204620361328,
+      "step": 2300
+    },
+    {
+      "epoch": 0.9870839021316812,
+      "grad_norm": 41.638790130615234,
+      "learning_rate": 0.00016250563985639204,
+      "loss": 1.8278160095214844,
+      "step": 2350
+    },
+    {
+      "epoch": 1.0079806783576604,
+      "grad_norm": 11.446990013122559,
+      "learning_rate": 0.00016066246944082394,
+      "loss": 1.7912734985351562,
+      "step": 2400
+    },
+    {
+      "epoch": 1.0289824635093983,
+      "grad_norm": 5.880982398986816,
+      "learning_rate": 0.00015878609602292562,
+      "loss": 1.458508758544922,
+      "step": 2450
+    },
+    {
+      "epoch": 1.0499842486611362,
+      "grad_norm": 99.73579406738281,
+      "learning_rate": 0.00015687754661711866,
+      "loss": 1.5889370727539063,
+      "step": 2500
+    },
+    {
+      "epoch": 1.070986033812874,
+      "grad_norm": 30.614892959594727,
+      "learning_rate": 0.0001549378658490333,
+      "loss": 1.396112823486328,
+      "step": 2550
+    },
+    {
+      "epoch": 1.091987818964612,
+      "grad_norm": 42.81917953491211,
+      "learning_rate": 0.00015296811538374278,
+      "loss": 1.2478562927246093,
+      "step": 2600
+    },
+    {
+      "epoch": 1.1129896041163498,
+      "grad_norm": 89.29583740234375,
+      "learning_rate": 0.00015096937334467151,
+      "loss": 1.6881478881835938,
+      "step": 2650
+    },
+    {
+      "epoch": 1.1339913892680877,
+      "grad_norm": 45.66031265258789,
+      "learning_rate": 0.000148942733723495,
+      "loss": 1.3388128662109375,
+      "step": 2700
+    },
+    {
+      "epoch": 1.1549931744198256,
+      "grad_norm": 19.098651885986328,
+      "learning_rate": 0.00014688930578135416,
+      "loss": 1.4715719604492188,
+      "step": 2750
+    },
+    {
+      "epoch": 1.1759949595715635,
+      "grad_norm": 40.75487518310547,
+      "learning_rate": 0.0001448102134417123,
+      "loss": 1.3008560180664062,
+      "step": 2800
+    },
+    {
+      "epoch": 1.1969967447233014,
+      "grad_norm": NaN,
+      "learning_rate": 0.00014270659467518682,
+      "loss": 1.458815155029297,
+      "step": 2850
+    },
+    {
+      "epoch": 1.2179985298750393,
+      "grad_norm": 59.06400680541992,
+      "learning_rate": 0.00014062236231847836,
+      "loss": 1.4431008911132812,
+      "step": 2900
+    },
+    {
+      "epoch": 1.2390003150267772,
+      "grad_norm": 55.12430953979492,
+      "learning_rate": 0.00013847359040513967,
+      "loss": 1.2680451965332031,
+      "step": 2950
+    },
+    {
+      "epoch": 1.260002100178515,
+      "grad_norm": 18.353561401367188,
+      "learning_rate": 0.0001363037603528289,
+      "loss": 1.3490068054199218,
+      "step": 3000
+    },
+    {
+      "epoch": 1.281003885330253,
+      "grad_norm": 46.3448371887207,
+      "learning_rate": 0.00013411405979655243,
+      "loss": 1.31285400390625,
+      "step": 3050
+    },
+    {
+      "epoch": 1.302005670481991,
+      "grad_norm": 28.819128036499023,
+      "learning_rate": 0.0001319056872472404,
+      "loss": 1.2427201080322265,
+      "step": 3100
+    },
+    {
+      "epoch": 1.3230074556337288,
+      "grad_norm": 41.59355545043945,
+      "learning_rate": 0.00012967985143575356,
+      "loss": 1.488096923828125,
+      "step": 3150
+    },
+    {
+      "epoch": 1.3440092407854667,
+      "grad_norm": 21.26289939880371,
+      "learning_rate": 0.00012743777065129635,
+      "loss": 1.3544757080078125,
+      "step": 3200
+    },
+    {
+      "epoch": 1.3650110259372046,
+      "grad_norm": 42.516868591308594,
+      "learning_rate": 0.0001251806720745984,
+      "loss": 1.3172706604003905,
+      "step": 3250
+    },
+    {
+      "epoch": 1.3860128110889427,
+      "grad_norm": 51.99966812133789,
+      "learning_rate": 0.00012290979110622937,
+      "loss": 1.340508270263672,
+      "step": 3300
+    },
+    {
+      "epoch": 1.4070145962406806,
+      "grad_norm": 40.60024642944336,
+      "learning_rate": 0.00012062637069041456,
+      "loss": 1.2897941589355468,
+      "step": 3350
+    },
+    {
+      "epoch": 1.4280163813924185,
+      "grad_norm": 31.378459930419922,
+      "learning_rate": 0.0001183316606347217,
+      "loss": 1.5410649108886718,
+      "step": 3400
+    },
+    {
+      "epoch": 1.4490181665441564,
+      "grad_norm": 26.510839462280273,
+      "learning_rate": 0.00011602691692599127,
+      "loss": 1.2167890167236328,
+      "step": 3450
+    },
+    {
+      "epoch": 1.4700199516958943,
+      "grad_norm": 23.495378494262695,
+      "learning_rate": 0.00011371340104288442,
+      "loss": 1.42876708984375,
+      "step": 3500
+    },
+    {
+      "epoch": 1.4910217368476322,
+      "grad_norm": 28.301671981811523,
+      "learning_rate": 0.00011139237926542509,
+      "loss": 1.063271255493164,
+      "step": 3550
+    },
+    {
+      "epoch": 1.51202352199937,
+      "grad_norm": 21.21889877319336,
+      "learning_rate": 0.0001090651219819143,
+      "loss": 1.2190794372558593,
+      "step": 3600
+    },
+    {
+      "epoch": 1.533025307151108,
+      "grad_norm": 15.157039642333984,
+      "learning_rate": 0.00010673290299359551,
+      "loss": 1.2155840301513672,
+      "step": 3650
+    },
+    {
+      "epoch": 1.5540270923028459,
+      "grad_norm": 12.83481502532959,
+      "learning_rate": 0.00010439699881745222,
+      "loss": 1.2669831085205079,
+      "step": 3700
+    },
+    {
+      "epoch": 1.5750288774545838,
+      "grad_norm": 40.47445297241211,
+      "learning_rate": 0.00010205868798751908,
+      "loss": 1.189565658569336,
+      "step": 3750
+    },
+    {
+      "epoch": 1.5960306626063216,
+      "grad_norm": 47.97238540649414,
+      "learning_rate": 9.971925035508882e-05,
+      "loss": 1.3227517700195313,
+      "step": 3800
+    },
+    {
+      "epoch": 1.6170324477580595,
+      "grad_norm": 11.237785339355469,
+      "learning_rate": 9.737996638819842e-05,
+      "loss": 1.1511013793945313,
+      "step": 3850
+    },
+    {
+      "epoch": 1.6380342329097974,
+      "grad_norm": 27.457271575927734,
+      "learning_rate": 9.504211647077748e-05,
+      "loss": 1.1926652526855468,
+      "step": 3900
+    },
+    {
+      "epoch": 1.6590360180615353,
+      "grad_norm": 15.741539001464844,
+      "learning_rate": 9.270698020184285e-05,
+      "loss": 1.0482740020751953,
+      "step": 3950
+    },
+    {
+      "epoch": 1.6800378032132732,
+      "grad_norm": 7.643248558044434,
+      "learning_rate": 9.037583569512271e-05,
+      "loss": 1.2052240753173828,
+      "step": 4000
+    },
+    {
+      "epoch": 1.7010395883650111,
+      "grad_norm": 20.112401962280273,
+      "learning_rate": 8.804995887949373e-05,
+      "loss": 1.0017765808105468,
+      "step": 4050
+    },
+    {
+      "epoch": 1.722041373516749,
+      "grad_norm": 22.75946617126465,
+      "learning_rate": 8.573062280061399e-05,
+      "loss": 1.348891143798828,
+      "step": 4100
+    },
+    {
+      "epoch": 1.743043158668487,
+      "grad_norm": 47.762718200683594,
+      "learning_rate": 8.341909692413401e-05,
+      "loss": 1.1363021850585937,
+      "step": 4150
+    },
+    {
+      "epoch": 1.7640449438202248,
+      "grad_norm": 28.285919189453125,
+      "learning_rate": 8.111664644086733e-05,
+      "loss": 1.1882960510253906,
+      "step": 4200
+    },
+    {
+      "epoch": 1.7850467289719627,
+      "grad_norm": 6.3662214279174805,
+      "learning_rate": 7.882453157430086e-05,
+      "loss": 1.21821044921875,
+      "step": 4250
+    },
+    {
+      "epoch": 1.8060485141237006,
+      "grad_norm": 11.175128936767578,
+      "learning_rate": 7.654400689082387e-05,
+      "loss": 1.1443672180175781,
+      "step": 4300
+    },
+    {
+      "epoch": 1.8270502992754385,
+      "grad_norm": 7.028075218200684,
+      "learning_rate": 7.427632061305376e-05,
+      "loss": 1.1513980865478515,
+      "step": 4350
+    },
+    {
+      "epoch": 1.8480520844271764,
+      "grad_norm": 3.8552818298339844,
+      "learning_rate": 7.202271393663354e-05,
+      "loss": 1.1591477966308594,
+      "step": 4400
+    },
+    {
+      "epoch": 1.8690538695789143,
+      "grad_norm": 27.834590911865234,
+      "learning_rate": 6.978442035087579e-05,
+      "loss": 1.0166816711425781,
+      "step": 4450
+    },
+    {
+      "epoch": 1.8900556547306522,
+      "grad_norm": 34.69282913208008,
+      "learning_rate": 6.756266496362442e-05,
+      "loss": 1.1785438537597657,
+      "step": 4500
+    },
+    {
+      "epoch": 1.91105743988239,
+      "grad_norm": 34.50713348388672,
+      "learning_rate": 6.535866383070405e-05,
+      "loss": 0.9877433013916016,
+      "step": 4550
+    },
+    {
+      "epoch": 1.932059225034128,
+      "grad_norm": 28.763914108276367,
+      "learning_rate": 6.317362329032388e-05,
+      "loss": 0.9768619537353516,
+      "step": 4600
+    },
+    {
+      "epoch": 1.9530610101858659,
+      "grad_norm": 44.40330505371094,
+      "learning_rate": 6.1008739302800486e-05,
+      "loss": 1.2748094940185546,
+      "step": 4650
+    },
+    {
+      "epoch": 1.9740627953376038,
+      "grad_norm": 45.88819122314453,
+      "learning_rate": 5.886519679596063e-05,
+      "loss": 0.8840785980224609,
+      "step": 4700
+    },
+    {
+      "epoch": 1.9950645804893417,
+      "grad_norm": 26.39780044555664,
+      "learning_rate": 5.674416901658288e-05,
+      "loss": 1.1054458618164062,
+      "step": 4750
+    },
+    {
+      "epoch": 2.0159613567153207,
+      "grad_norm": 28.105772018432617,
+      "learning_rate": 5.464681688823246e-05,
+      "loss": 0.43650150299072266,
+      "step": 4800
+    },
+    {
+      "epoch": 2.0369631418670586,
+      "grad_norm": 0.8811307549476624,
+      "learning_rate": 5.257428837584142e-05,
+      "loss": 0.4781087112426758,
+      "step": 4850
+    },
+    {
+      "epoch": 2.0579649270187965,
+      "grad_norm": 29.04726791381836,
+      "learning_rate": 5.052771785738109e-05,
+      "loss": 0.538114242553711,
+      "step": 4900
+    },
+    {
+      "epoch": 2.0789667121705344,
+      "grad_norm": 2.272526741027832,
+      "learning_rate": 4.8548342809611756e-05,
+      "loss": 0.5318920516967773,
+      "step": 4950
+    },
+    {
+      "epoch": 2.0999684973222723,
+      "grad_norm": 35.368194580078125,
+      "learning_rate": 4.655645956450846e-05,
+      "loss": 0.48801342010498044,
+      "step": 5000
+    },
+    {
+      "epoch": 2.12097028247401,
+      "grad_norm": 0.03809299319982529,
+      "learning_rate": 4.4593828112383176e-05,
+      "loss": 0.4779471206665039,
+      "step": 5050
+    },
+    {
+      "epoch": 2.141972067625748,
+      "grad_norm": 8.704736709594727,
+      "learning_rate": 4.269986472510045e-05,
+      "loss": 0.6272232437133789,
+      "step": 5100
+    },
+    {
+      "epoch": 2.162973852777486,
+      "grad_norm": 2.0720932483673096,
+      "learning_rate": 4.0798305013432315e-05,
+      "loss": 0.43107017517089846,
+      "step": 5150
+    },
+    {
+      "epoch": 2.183975637929224,
+      "grad_norm": 8.66932201385498,
+      "learning_rate": 3.8929148763652736e-05,
+      "loss": 0.5629381561279296,
+      "step": 5200
+    },
+    {
+      "epoch": 2.204977423080962,
+      "grad_norm": 36.49018096923828,
+      "learning_rate": 3.7093419039937825e-05,
+      "loss": 0.6745287322998047,
+      "step": 5250
+    },
+    {
+      "epoch": 2.2259792082326997,
+      "grad_norm": 44.782039642333984,
+      "learning_rate": 3.529212061078543e-05,
+      "loss": 0.44454833984375,
+      "step": 5300
+    },
+    {
+      "epoch": 2.2469809933844376,
+      "grad_norm": 38.312313079833984,
+      "learning_rate": 3.352623939906491e-05,
+      "loss": 0.4252283477783203,
+      "step": 5350
+    },
+    {
+      "epoch": 2.2679827785361755,
+      "grad_norm": 48.045509338378906,
+      "learning_rate": 3.179674194238237e-05,
+      "loss": 0.6742375183105469,
+      "step": 5400
+    },
+    {
+      "epoch": 2.2889845636879134,
+      "grad_norm": 63.853843688964844,
+      "learning_rate": 3.010457486405549e-05,
+      "loss": 0.4491603851318359,
+      "step": 5450
+    },
+    {
+      "epoch": 2.3099863488396513,
+      "grad_norm": 1.4884780645370483,
+      "learning_rate": 2.84506643549889e-05,
+      "loss": 0.5363286972045899,
+      "step": 5500
+    },
+    {
+      "epoch": 2.330988133991389,
+      "grad_norm": 0.02728688158094883,
+      "learning_rate": 2.6835915666732548e-05,
+      "loss": 0.34341278076171877,
+      "step": 5550
+    },
+    {
+      "epoch": 2.351989919143127,
+      "grad_norm": 46.706119537353516,
+      "learning_rate": 2.5261212616001327e-05,
+      "loss": 0.6819667816162109,
+      "step": 5600
+    },
+    {
+      "epoch": 2.372991704294865,
+      "grad_norm": 2.9122889041900635,
+      "learning_rate": 2.372741710092674e-05,
+      "loss": 0.3454324722290039,
+      "step": 5650
+    },
+    {
+      "epoch": 2.393993489446603,
+      "grad_norm": 2.8348746299743652,
+      "learning_rate": 2.2235368629305418e-05,
+      "loss": 0.37347347259521485,
+      "step": 5700
+    },
+    {
+      "epoch": 2.4149952745983407,
+      "grad_norm": 2.396700859069824,
+      "learning_rate": 2.0785883859102994e-05,
+      "loss": 0.478424072265625,
+      "step": 5750
+    },
+    {
+      "epoch": 2.4359970597500786,
+      "grad_norm": 0.3374341130256653,
+      "learning_rate": 1.9379756151464236e-05,
+      "loss": 0.3879497528076172,
+      "step": 5800
+    },
+    {
+      "epoch": 2.4569988449018165,
+      "grad_norm": 0.07735343277454376,
+      "learning_rate": 1.801775513647487e-05,
+      "loss": 0.3089980697631836,
+      "step": 5850
+    },
+    {
+      "epoch": 2.4780006300535544,
+      "grad_norm": 45.71025848388672,
+      "learning_rate": 1.6700626291912004e-05,
+      "loss": 0.3591925048828125,
+      "step": 5900
+    },
+    {
+      "epoch": 2.4990024152052923,
+      "grad_norm": 86.34896850585938,
+      "learning_rate": 1.5429090535214353e-05,
+      "loss": 0.4602979278564453,
+      "step": 5950
+    },
+    {
+      "epoch": 2.52000420035703,
+      "grad_norm": 125.41795349121094,
+      "learning_rate": 1.420384382889518e-05,
+      "loss": 0.4123740005493164,
+      "step": 6000
+    },
+    {
+      "epoch": 2.541005985508768,
+      "grad_norm": 0.06989021599292755,
+      "learning_rate": 1.302555679961408e-05,
+      "loss": 0.5613171768188476,
+      "step": 6050
+    },
+    {
+      "epoch": 2.562007770660506,
+      "grad_norm": 50.419254302978516,
+      "learning_rate": 1.1894874371116182e-05,
+      "loss": 0.44726715087890623,
+      "step": 6100
+    },
+    {
+      "epoch": 2.583009555812244,
+      "grad_norm": 29.774009704589844,
+      "learning_rate": 1.081241541123944e-05,
+      "loss": 0.36859161376953126,
+      "step": 6150
+    },
+    {
+      "epoch": 2.604011340963982,
+      "grad_norm": 154.20655822753906,
+      "learning_rate": 9.778772393183467e-06,
+      "loss": 0.5629509735107422,
+      "step": 6200
+    },
+    {
+      "epoch": 2.6250131261157197,
+      "grad_norm": 0.09836237877607346,
+      "learning_rate": 8.794511071225275e-06,
+      "loss": 0.371593132019043,
+      "step": 6250
+    },
+    {
+      "epoch": 2.6460149112674576,
+      "grad_norm": 56.41450500488281,
+      "learning_rate": 7.860170171059067e-06,
+      "loss": 0.44127925872802737,
+      "step": 6300
+    },
+    {
+      "epoch": 2.6670166964191955,
+      "grad_norm": 30.18195152282715,
+      "learning_rate": 6.976261094930347e-06,
+      "loss": 0.5457481384277344,
+      "step": 6350
+    },
+    {
+      "epoch": 2.6880184815709334,
+      "grad_norm": 1.9519455432891846,
+      "learning_rate": 6.1432676417246925e-06,
+      "loss": 0.46905494689941407,
+      "step": 6400
+    },
+    {
+      "epoch": 2.7090202667226713,
+      "grad_norm": 0.3305146396160126,
+      "learning_rate": 5.3616457421654e-06,
+      "loss": 0.18893402099609374,
+      "step": 6450
+    },
+    {
+      "epoch": 2.730022051874409,
+      "grad_norm": 0.018915779888629913,
+      "learning_rate": 4.63182320926423e-06,
+      "loss": 0.5251737594604492,
+      "step": 6500
+    },
+    {
+      "epoch": 2.751023837026147,
+      "grad_norm": 2.023514986038208,
+      "learning_rate": 3.9541995041621925e-06,
+      "loss": 0.3400577163696289,
+      "step": 6550
+    },
+    {
+      "epoch": 2.7720256221778854,
+      "grad_norm": 27.531789779663086,
+      "learning_rate": 3.32914551748853e-06,
+      "loss": 0.4619370269775391,
+      "step": 6600
+    },
+    {
+      "epoch": 2.793027407329623,
+      "grad_norm": 23.563621520996094,
+      "learning_rate": 2.757003366357236e-06,
+      "loss": 0.3556163787841797,
+      "step": 6650
+    },
+    {
+      "epoch": 2.814029192481361,
+      "grad_norm": 0.1751919537782669,
+      "learning_rate": 2.2380862071128327e-06,
+      "loss": 0.43869766235351565,
+      "step": 6700
+    },
+    {
+      "epoch": 2.8350309776330986,
+      "grad_norm": 74.48571014404297,
+      "learning_rate": 1.77267806392718e-06,
+      "loss": 0.43838497161865236,
+      "step": 6750
+    },
+    {
+      "epoch": 2.856032762784837,
+      "grad_norm": 0.02355477400124073,
+      "learning_rate": 1.3610336733417605e-06,
+      "loss": 0.27370246887207034,
+      "step": 6800
+    },
+    {
+      "epoch": 2.8770345479365744,
+      "grad_norm": 79.37924194335938,
+      "learning_rate": 1.0033783448401112e-06,
+      "loss": 0.521487808227539,
+      "step": 6850
+    },
+    {
+      "epoch": 2.8980363330883128,
+      "grad_norm": 0.01911379024386406,
+      "learning_rate": 6.999078375269252e-07,
+      "loss": 0.424820671081543,
+      "step": 6900
+    },
+    {
+      "epoch": 2.91903811824005,
+      "grad_norm": 63.20115661621094,
+      "learning_rate": 4.5078825298122683e-07,
+      "loss": 0.48077041625976563,
+      "step": 6950
+    },
+    {
+      "epoch": 2.9400399033917886,
+      "grad_norm": 19.453731536865234,
+      "learning_rate": 2.5615594434229564e-07,
+      "loss": 0.4773090362548828,
+      "step": 7000
+    },
+    {
+      "epoch": 2.961041688543526,
+      "grad_norm": 0.6832577586174011,
+      "learning_rate": 1.161174416781985e-07,
+      "loss": 0.4181100082397461,
+      "step": 7050
+    },
+    {
+      "epoch": 2.9820434736952643,
+      "grad_norm": 2.216742992401123,
+      "learning_rate": 3.07493936775427e-08,
+      "loss": 0.252137451171875,
+      "step": 7100
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 7143,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 6.162234131538248e+17,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-7143/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3dcfe9cb7f80cca256d11b427979266d513b9e02e5d47123e2e934069b9ff0d4
+size 5265

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbe07b35a61be0ec850f12e9379be8ca47b4ef845b20f0d146a8a2bfdb0e7e3e
+size 17210195

tokenizer_config.json

@@ -0,0 +1,14 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "is_local": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|eot_id|>",
+  "tokenizer_class": "TokenizersBackend"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3dcfe9cb7f80cca256d11b427979266d513b9e02e5d47123e2e934069b9ff0d4
+size 5265