Duplicate from aleegis12/7b8a69b7-8a9b-42af-bd11-6d1906937c96

Co-authored-by: AleeGis 12 <[email protected]>

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,168 @@
+---
+library_name: peft
+license: other
+base_model: facebook/opt-125m
+tags:
+- axolotl
+- generated_from_trainer
+model-index:
+- name: 7b8a69b7-8a9b-42af-bd11-6d1906937c96
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/axolotl-ai-cloud/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/axolotl-ai-cloud/axolotl)
+<details><summary>See axolotl config</summary>
+
+axolotl version: `0.4.1`
+```yaml
+adapter: lora
+base_model: facebook/opt-125m
+bf16: true
+chat_template: llama3
+dataset_prepared_path: null
+datasets:
+- data_files:
+  - 410376cf4dc313b6_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/410376cf4dc313b6_train_data.json
+  type:
+    field_instruction: article
+    field_output: highlights
+    format: '{instruction}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+debug: null
+deepspeed: null
+device_map: auto
+do_eval: true
+early_stopping_patience: 5
+eval_batch_size: 4
+eval_max_new_tokens: 128
+eval_steps: 50
+eval_table_size: null
+evals_per_epoch: null
+flash_attention: true
+fp16: false
+fsdp: null
+fsdp_config: null
+gradient_accumulation_steps: 4
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: aleegis12/7b8a69b7-8a9b-42af-bd11-6d1906937c96
+hub_repo: null
+hub_strategy: checkpoint
+hub_token: null
+learning_rate: 0.0001
+load_in_4bit: false
+load_in_8bit: false
+local_rank: null
+logging_steps: 1
+lora_alpha: 128
+lora_dropout: 0.05
+lora_fan_in_fan_out: null
+lora_model_dir: null
+lora_r: 64
+lora_target_linear: true
+lr_scheduler: cosine
+max_grad_norm: 1.0
+max_memory:
+  0: 75GB
+max_steps: 400
+micro_batch_size: 8
+mlflow_experiment_name: /tmp/410376cf4dc313b6_train_data.json
+model_type: AutoModelForCausalLM
+num_epochs: 1
+optim_args:
+  adam_beta1: 0.9
+  adam_beta2: 0.95
+  adam_epsilon: 1e-5
+optimizer: adamw_bnb_8bit
+output_dir: miner_id_24
+pad_to_sequence_len: true
+resume_from_checkpoint: null
+s2_attention: null
+sample_packing: false
+save_steps: 50
+saves_per_epoch: null
+sequence_len: 1024
+strict: false
+tf32: true
+tokenizer_type: AutoTokenizer
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.05
+wandb_entity: null
+wandb_mode: online
+wandb_name: aef86b87-9f89-4e92-86bf-4d67dc00f490
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: aef86b87-9f89-4e92-86bf-4d67dc00f490
+warmup_steps: 10
+weight_decay: 0.0
+xformers_attention: null
+
+```
+
+</details><br>
+
+# 7b8a69b7-8a9b-42af-bd11-6d1906937c96
+
+This model is a fine-tuned version of [facebook/opt-125m](https://huggingface.co/facebook/opt-125m) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 2.0413
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0001
+- train_batch_size: 8
+- eval_batch_size: 4
+- seed: 42
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 32
+- optimizer: Use OptimizerNames.ADAMW_BNB with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=adam_beta1=0.9,adam_beta2=0.95,adam_epsilon=1e-5
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_steps: 10
+- training_steps: 400
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss |
+|:-------------:|:------:|:----:|:---------------:|
+| 9.9392        | 0.0001 | 1    | 2.7665          |
+| 8.3319        | 0.0054 | 50   | 2.2252          |
+| 7.4759        | 0.0108 | 100  | 2.1500          |
+| 7.826         | 0.0162 | 150  | 2.1161          |
+| 7.5391        | 0.0216 | 200  | 2.1373          |
+| 7.5569        | 0.0270 | 250  | 2.0691          |
+| 7.5241        | 0.0324 | 300  | 2.0537          |
+| 8.0514        | 0.0378 | 350  | 2.0434          |
+| 7.131         | 0.0432 | 400  | 2.0413          |
+
+
+### Framework versions
+
+- PEFT 0.13.2
+- Transformers 4.46.0
+- Pytorch 2.5.0+cu124
+- Datasets 3.0.1
+- Tokenizers 0.20.1

adapter_config.json

@@ -0,0 +1,33 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "facebook/opt-125m",
+  "bias": "none",
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 128,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "fc2",
+    "v_proj",
+    "fc1",
+    "k_proj",
+    "q_proj",
+    "out_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6de0e47ba53e032c7f72c5fe4203009b21100bdae016d2c05d34308f6c44f579
+size 42519882

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b841195cb259215de6c9c953c4ad0468a0c5236bdcd0c82425ba513b983b9d32
+size 42487072

config.json

@@ -0,0 +1,32 @@
+{
+  "_attn_implementation_autoset": true,
+  "_name_or_path": "facebook/opt-125m",
+  "_remove_final_layer_norm": false,
+  "activation_dropout": 0.0,
+  "activation_function": "relu",
+  "architectures": [
+    "OPTForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 2,
+  "do_layer_norm_before": true,
+  "dropout": 0.1,
+  "enable_bias": true,
+  "eos_token_id": 2,
+  "ffn_dim": 3072,
+  "hidden_size": 768,
+  "init_std": 0.02,
+  "layer_norm_elementwise_affine": true,
+  "layerdrop": 0.0,
+  "max_position_embeddings": 2048,
+  "model_type": "opt",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "prefix": "</s>",
+  "torch_dtype": "float16",
+  "transformers_version": "4.46.0",
+  "use_cache": false,
+  "vocab_size": 50272,
+  "word_embed_proj_dim": 768
+}

last-checkpoint/README.md

@@ -0,0 +1,202 @@
+---
+base_model: facebook/opt-125m
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.13.2

last-checkpoint/adapter_config.json

@@ -0,0 +1,33 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "facebook/opt-125m",
+  "bias": "none",
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 128,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "fc2",
+    "v_proj",
+    "fc1",
+    "k_proj",
+    "q_proj",
+    "out_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

last-checkpoint/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b841195cb259215de6c9c953c4ad0468a0c5236bdcd0c82425ba513b983b9d32
+size 42487072

last-checkpoint/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c0ba5d3cc1cff3a93dc96e92a5b90c4f26afcd2e8c035def92d398f309402d8
+size 21735482

last-checkpoint/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5fa68317afd576605320f44f013cf246e87d453b2b2b27bce7138dd1fe2ca74
+size 14244

last-checkpoint/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:705cabf5cbc3a6ab0feb67c77b9b453d59efcc939ce90d310af96e621810f990
+size 1064

last-checkpoint/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

last-checkpoint/tokenizer_config.json

@@ -0,0 +1,31 @@
+{
+  "add_bos_token": true,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "</s>",
+  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "errors": "replace",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "</s>"
+}

last-checkpoint/trainer_state.json

@@ -0,0 +1,2914 @@
+{
+  "best_metric": 2.0413002967834473,
+  "best_model_checkpoint": "miner_id_24/checkpoint-400",
+  "epoch": 0.0432163790076439,
+  "eval_steps": 50,
+  "global_step": 400,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.00010804094751910974,
+      "grad_norm": 10.902341842651367,
+      "learning_rate": 1e-05,
+      "loss": 9.9392,
+      "step": 1
+    },
+    {
+      "epoch": 0.00010804094751910974,
+      "eval_loss": 2.766528844833374,
+      "eval_runtime": 115.0025,
+      "eval_samples_per_second": 135.554,
+      "eval_steps_per_second": 33.895,
+      "step": 1
+    },
+    {
+      "epoch": 0.00021608189503821948,
+      "grad_norm": 14.739710807800293,
+      "learning_rate": 2e-05,
+      "loss": 10.4142,
+      "step": 2
+    },
+    {
+      "epoch": 0.00032412284255732923,
+      "grad_norm": 15.09880256652832,
+      "learning_rate": 3e-05,
+      "loss": 10.3479,
+      "step": 3
+    },
+    {
+      "epoch": 0.00043216379007643895,
+      "grad_norm": 14.444519996643066,
+      "learning_rate": 4e-05,
+      "loss": 10.6141,
+      "step": 4
+    },
+    {
+      "epoch": 0.0005402047375955487,
+      "grad_norm": 15.361382484436035,
+      "learning_rate": 5e-05,
+      "loss": 11.5047,
+      "step": 5
+    },
+    {
+      "epoch": 0.0006482456851146585,
+      "grad_norm": 13.631402969360352,
+      "learning_rate": 6e-05,
+      "loss": 11.4629,
+      "step": 6
+    },
+    {
+      "epoch": 0.0007562866326337682,
+      "grad_norm": 11.211657524108887,
+      "learning_rate": 7e-05,
+      "loss": 11.5248,
+      "step": 7
+    },
+    {
+      "epoch": 0.0008643275801528779,
+      "grad_norm": 10.227496147155762,
+      "learning_rate": 8e-05,
+      "loss": 11.1193,
+      "step": 8
+    },
+    {
+      "epoch": 0.0009723685276719876,
+      "grad_norm": 11.774316787719727,
+      "learning_rate": 9e-05,
+      "loss": 11.5137,
+      "step": 9
+    },
+    {
+      "epoch": 0.0010804094751910975,
+      "grad_norm": 11.15668773651123,
+      "learning_rate": 0.0001,
+      "loss": 11.0709,
+      "step": 10
+    },
+    {
+      "epoch": 0.001188450422710207,
+      "grad_norm": 12.9327392578125,
+      "learning_rate": 9.99983777858264e-05,
+      "loss": 11.8218,
+      "step": 11
+    },
+    {
+      "epoch": 0.001296491370229317,
+      "grad_norm": 11.049345016479492,
+      "learning_rate": 9.999351124856874e-05,
+      "loss": 10.8186,
+      "step": 12
+    },
+    {
+      "epoch": 0.0014045323177484267,
+      "grad_norm": 11.759522438049316,
+      "learning_rate": 9.998540070400966e-05,
+      "loss": 11.5046,
+      "step": 13
+    },
+    {
+      "epoch": 0.0015125732652675364,
+      "grad_norm": 12.516517639160156,
+      "learning_rate": 9.997404667843075e-05,
+      "loss": 10.8009,
+      "step": 14
+    },
+    {
+      "epoch": 0.0016206142127866462,
+      "grad_norm": 11.518903732299805,
+      "learning_rate": 9.995944990857849e-05,
+      "loss": 10.9571,
+      "step": 15
+    },
+    {
+      "epoch": 0.0017286551603057558,
+      "grad_norm": 10.633302688598633,
+      "learning_rate": 9.994161134161634e-05,
+      "loss": 11.1949,
+      "step": 16
+    },
+    {
+      "epoch": 0.0018366961078248656,
+      "grad_norm": 11.844691276550293,
+      "learning_rate": 9.992053213506334e-05,
+      "loss": 11.0355,
+      "step": 17
+    },
+    {
+      "epoch": 0.0019447370553439753,
+      "grad_norm": 9.481794357299805,
+      "learning_rate": 9.989621365671902e-05,
+      "loss": 9.7961,
+      "step": 18
+    },
+    {
+      "epoch": 0.002052778002863085,
+      "grad_norm": 9.508196830749512,
+      "learning_rate": 9.986865748457457e-05,
+      "loss": 9.9319,
+      "step": 19
+    },
+    {
+      "epoch": 0.002160818950382195,
+      "grad_norm": 8.430398941040039,
+      "learning_rate": 9.983786540671051e-05,
+      "loss": 8.7856,
+      "step": 20
+    },
+    {
+      "epoch": 0.0022688598979013048,
+      "grad_norm": 7.660574436187744,
+      "learning_rate": 9.980383942118066e-05,
+      "loss": 9.7413,
+      "step": 21
+    },
+    {
+      "epoch": 0.002376900845420414,
+      "grad_norm": 7.800100803375244,
+      "learning_rate": 9.976658173588244e-05,
+      "loss": 9.4984,
+      "step": 22
+    },
+    {
+      "epoch": 0.002484941792939524,
+      "grad_norm": 8.076692581176758,
+      "learning_rate": 9.972609476841367e-05,
+      "loss": 9.323,
+      "step": 23
+    },
+    {
+      "epoch": 0.002592982740458634,
+      "grad_norm": 7.220234394073486,
+      "learning_rate": 9.968238114591566e-05,
+      "loss": 8.7967,
+      "step": 24
+    },
+    {
+      "epoch": 0.0027010236879777437,
+      "grad_norm": 7.806915760040283,
+      "learning_rate": 9.96354437049027e-05,
+      "loss": 10.0444,
+      "step": 25
+    },
+    {
+      "epoch": 0.0028090646354968535,
+      "grad_norm": 7.543822765350342,
+      "learning_rate": 9.95852854910781e-05,
+      "loss": 8.131,
+      "step": 26
+    },
+    {
+      "epoch": 0.002917105583015963,
+      "grad_norm": 8.648165702819824,
+      "learning_rate": 9.953190975913647e-05,
+      "loss": 9.2058,
+      "step": 27
+    },
+    {
+      "epoch": 0.0030251465305350727,
+      "grad_norm": 7.434932708740234,
+      "learning_rate": 9.947531997255256e-05,
+      "loss": 8.3371,
+      "step": 28
+    },
+    {
+      "epoch": 0.0031331874780541826,
+      "grad_norm": 7.321508407592773,
+      "learning_rate": 9.941551980335652e-05,
+      "loss": 8.2415,
+      "step": 29
+    },
+    {
+      "epoch": 0.0032412284255732924,
+      "grad_norm": 7.649103164672852,
+      "learning_rate": 9.935251313189564e-05,
+      "loss": 8.6599,
+      "step": 30
+    },
+    {
+      "epoch": 0.0033492693730924022,
+      "grad_norm": 7.984175682067871,
+      "learning_rate": 9.928630404658255e-05,
+      "loss": 8.6025,
+      "step": 31
+    },
+    {
+      "epoch": 0.0034573103206115116,
+      "grad_norm": 7.4873504638671875,
+      "learning_rate": 9.921689684362989e-05,
+      "loss": 8.3759,
+      "step": 32
+    },
+    {
+      "epoch": 0.0035653512681306215,
+      "grad_norm": 7.798126697540283,
+      "learning_rate": 9.914429602677162e-05,
+      "loss": 8.3052,
+      "step": 33
+    },
+    {
+      "epoch": 0.0036733922156497313,
+      "grad_norm": 9.170891761779785,
+      "learning_rate": 9.906850630697068e-05,
+      "loss": 9.2459,
+      "step": 34
+    },
+    {
+      "epoch": 0.003781433163168841,
+      "grad_norm": 7.661440372467041,
+      "learning_rate": 9.898953260211338e-05,
+      "loss": 8.6468,
+      "step": 35
+    },
+    {
+      "epoch": 0.0038894741106879505,
+      "grad_norm": 7.861063480377197,
+      "learning_rate": 9.890738003669029e-05,
+      "loss": 8.3884,
+      "step": 36
+    },
+    {
+      "epoch": 0.003997515058207061,
+      "grad_norm": 7.5660834312438965,
+      "learning_rate": 9.882205394146361e-05,
+      "loss": 9.1711,
+      "step": 37
+    },
+    {
+      "epoch": 0.00410555600572617,
+      "grad_norm": 8.354244232177734,
+      "learning_rate": 9.87335598531214e-05,
+      "loss": 9.0446,
+      "step": 38
+    },
+    {
+      "epoch": 0.00421359695324528,
+      "grad_norm": 8.231657028198242,
+      "learning_rate": 9.864190351391822e-05,
+      "loss": 8.3486,
+      "step": 39
+    },
+    {
+      "epoch": 0.00432163790076439,
+      "grad_norm": 8.189095497131348,
+      "learning_rate": 9.85470908713026e-05,
+      "loss": 8.9636,
+      "step": 40
+    },
+    {
+      "epoch": 0.004429678848283499,
+      "grad_norm": 7.741812229156494,
+      "learning_rate": 9.844912807753104e-05,
+      "loss": 7.9608,
+      "step": 41
+    },
+    {
+      "epoch": 0.0045377197958026095,
+      "grad_norm": 8.20760440826416,
+      "learning_rate": 9.834802148926882e-05,
+      "loss": 8.7542,
+      "step": 42
+    },
+    {
+      "epoch": 0.004645760743321719,
+      "grad_norm": 8.086214065551758,
+      "learning_rate": 9.824377766717759e-05,
+      "loss": 8.9794,
+      "step": 43
+    },
+    {
+      "epoch": 0.004753801690840828,
+      "grad_norm": 8.539205551147461,
+      "learning_rate": 9.813640337548954e-05,
+      "loss": 8.159,
+      "step": 44
+    },
+    {
+      "epoch": 0.004861842638359939,
+      "grad_norm": 8.882444381713867,
+      "learning_rate": 9.802590558156862e-05,
+      "loss": 8.8944,
+      "step": 45
+    },
+    {
+      "epoch": 0.004969883585879048,
+      "grad_norm": 7.916795253753662,
+      "learning_rate": 9.791229145545831e-05,
+      "loss": 8.9551,
+      "step": 46
+    },
+    {
+      "epoch": 0.005077924533398158,
+      "grad_norm": 7.607052326202393,
+      "learning_rate": 9.779556836941645e-05,
+      "loss": 7.6425,
+      "step": 47
+    },
+    {
+      "epoch": 0.005185965480917268,
+      "grad_norm": 7.65380859375,
+      "learning_rate": 9.767574389743682e-05,
+      "loss": 7.6339,
+      "step": 48
+    },
+    {
+      "epoch": 0.005294006428436377,
+      "grad_norm": 7.760143756866455,
+      "learning_rate": 9.755282581475769e-05,
+      "loss": 7.7469,
+      "step": 49
+    },
+    {
+      "epoch": 0.005402047375955487,
+      "grad_norm": 8.861435890197754,
+      "learning_rate": 9.742682209735727e-05,
+      "loss": 8.3319,
+      "step": 50
+    },
+    {
+      "epoch": 0.005402047375955487,
+      "eval_loss": 2.2252063751220703,
+      "eval_runtime": 115.0214,
+      "eval_samples_per_second": 135.531,
+      "eval_steps_per_second": 33.889,
+      "step": 50
+    },
+    {
+      "epoch": 0.005510088323474597,
+      "grad_norm": 12.5848970413208,
+      "learning_rate": 9.729774092143627e-05,
+      "loss": 10.2893,
+      "step": 51
+    },
+    {
+      "epoch": 0.005618129270993707,
+      "grad_norm": 10.760187149047852,
+      "learning_rate": 9.716559066288715e-05,
+      "loss": 9.7445,
+      "step": 52
+    },
+    {
+      "epoch": 0.005726170218512816,
+      "grad_norm": 8.755156517028809,
+      "learning_rate": 9.703037989675087e-05,
+      "loss": 9.4308,
+      "step": 53
+    },
+    {
+      "epoch": 0.005834211166031926,
+      "grad_norm": 9.094099044799805,
+      "learning_rate": 9.689211739666023e-05,
+      "loss": 9.6687,
+      "step": 54
+    },
+    {
+      "epoch": 0.005942252113551036,
+      "grad_norm": 8.874935150146484,
+      "learning_rate": 9.675081213427076e-05,
+      "loss": 10.3187,
+      "step": 55
+    },
+    {
+      "epoch": 0.0060502930610701455,
+      "grad_norm": 8.958069801330566,
+      "learning_rate": 9.66064732786784e-05,
+      "loss": 9.3601,
+      "step": 56
+    },
+    {
+      "epoch": 0.006158334008589256,
+      "grad_norm": 9.862615585327148,
+      "learning_rate": 9.645911019582467e-05,
+      "loss": 10.9413,
+      "step": 57
+    },
+    {
+      "epoch": 0.006266374956108365,
+      "grad_norm": 8.985273361206055,
+      "learning_rate": 9.630873244788883e-05,
+      "loss": 10.7751,
+      "step": 58
+    },
+    {
+      "epoch": 0.0063744159036274745,
+      "grad_norm": 8.574797630310059,
+      "learning_rate": 9.615534979266745e-05,
+      "loss": 10.8228,
+      "step": 59
+    },
+    {
+      "epoch": 0.006482456851146585,
+      "grad_norm": 7.84227180480957,
+      "learning_rate": 9.599897218294122e-05,
+      "loss": 10.7216,
+      "step": 60
+    },
+    {
+      "epoch": 0.006590497798665694,
+      "grad_norm": 7.276218414306641,
+      "learning_rate": 9.583960976582913e-05,
+      "loss": 10.6069,
+      "step": 61
+    },
+    {
+      "epoch": 0.0066985387461848045,
+      "grad_norm": 7.8485541343688965,
+      "learning_rate": 9.567727288213005e-05,
+      "loss": 9.8764,
+      "step": 62
+    },
+    {
+      "epoch": 0.006806579693703914,
+      "grad_norm": 8.47364330291748,
+      "learning_rate": 9.551197206565173e-05,
+      "loss": 10.3013,
+      "step": 63
+    },
+    {
+      "epoch": 0.006914620641223023,
+      "grad_norm": 8.667262077331543,
+      "learning_rate": 9.534371804252728e-05,
+      "loss": 10.0584,
+      "step": 64
+    },
+    {
+      "epoch": 0.0070226615887421335,
+      "grad_norm": 8.350459098815918,
+      "learning_rate": 9.517252173051911e-05,
+      "loss": 10.7361,
+      "step": 65
+    },
+    {
+      "epoch": 0.007130702536261243,
+      "grad_norm": 8.560527801513672,
+      "learning_rate": 9.49983942383106e-05,
+      "loss": 10.5065,
+      "step": 66
+    },
+    {
+      "epoch": 0.007238743483780352,
+      "grad_norm": 8.592406272888184,
+      "learning_rate": 9.482134686478519e-05,
+      "loss": 9.277,
+      "step": 67
+    },
+    {
+      "epoch": 0.007346784431299463,
+      "grad_norm": 7.4438371658325195,
+      "learning_rate": 9.464139109829321e-05,
+      "loss": 9.0273,
+      "step": 68
+    },
+    {
+      "epoch": 0.007454825378818572,
+      "grad_norm": 7.362355709075928,
+      "learning_rate": 9.445853861590647e-05,
+      "loss": 9.2297,
+      "step": 69
+    },
+    {
+      "epoch": 0.007562866326337682,
+      "grad_norm": 7.441712856292725,
+      "learning_rate": 9.42728012826605e-05,
+      "loss": 8.6039,
+      "step": 70
+    },
+    {
+      "epoch": 0.007670907273856792,
+      "grad_norm": 7.506527900695801,
+      "learning_rate": 9.408419115078471e-05,
+      "loss": 9.0215,
+      "step": 71
+    },
+    {
+      "epoch": 0.007778948221375901,
+      "grad_norm": 7.2407026290893555,
+      "learning_rate": 9.389272045892024e-05,
+      "loss": 8.4238,
+      "step": 72
+    },
+    {
+      "epoch": 0.007886989168895011,
+      "grad_norm": 7.309229850769043,
+      "learning_rate": 9.36984016313259e-05,
+      "loss": 9.5363,
+      "step": 73
+    },
+    {
+      "epoch": 0.007995030116414122,
+      "grad_norm": 7.002497673034668,
+      "learning_rate": 9.350124727707197e-05,
+      "loss": 8.3292,
+      "step": 74
+    },
+    {
+      "epoch": 0.00810307106393323,
+      "grad_norm": 7.012520790100098,
+      "learning_rate": 9.330127018922194e-05,
+      "loss": 8.1964,
+      "step": 75
+    },
+    {
+      "epoch": 0.00821111201145234,
+      "grad_norm": 6.851153373718262,
+      "learning_rate": 9.309848334400246e-05,
+      "loss": 8.5043,
+      "step": 76
+    },
+    {
+      "epoch": 0.00831915295897145,
+      "grad_norm": 7.714682102203369,
+      "learning_rate": 9.289289989996133e-05,
+      "loss": 8.7056,
+      "step": 77
+    },
+    {
+      "epoch": 0.00842719390649056,
+      "grad_norm": 7.203169345855713,
+      "learning_rate": 9.268453319711363e-05,
+      "loss": 8.5106,
+      "step": 78
+    },
+    {
+      "epoch": 0.00853523485400967,
+      "grad_norm": 7.153778553009033,
+      "learning_rate": 9.247339675607605e-05,
+      "loss": 9.3719,
+      "step": 79
+    },
+    {
+      "epoch": 0.00864327580152878,
+      "grad_norm": 6.689031600952148,
+      "learning_rate": 9.225950427718975e-05,
+      "loss": 8.2862,
+      "step": 80
+    },
+    {
+      "epoch": 0.00875131674904789,
+      "grad_norm": 6.7153096199035645,
+      "learning_rate": 9.204286963963111e-05,
+      "loss": 8.5915,
+      "step": 81
+    },
+    {
+      "epoch": 0.008859357696566999,
+      "grad_norm": 6.7668776512146,
+      "learning_rate": 9.182350690051133e-05,
+      "loss": 8.4989,
+      "step": 82
+    },
+    {
+      "epoch": 0.008967398644086109,
+      "grad_norm": 6.883266448974609,
+      "learning_rate": 9.160143029396422e-05,
+      "loss": 8.1489,
+      "step": 83
+    },
+    {
+      "epoch": 0.009075439591605219,
+      "grad_norm": 7.132114410400391,
+      "learning_rate": 9.13766542302225e-05,
+      "loss": 7.6873,
+      "step": 84
+    },
+    {
+      "epoch": 0.009183480539124328,
+      "grad_norm": 6.811685562133789,
+      "learning_rate": 9.114919329468282e-05,
+      "loss": 7.6089,
+      "step": 85
+    },
+    {
+      "epoch": 0.009291521486643438,
+      "grad_norm": 6.8911638259887695,
+      "learning_rate": 9.091906224695935e-05,
+      "loss": 8.0561,
+      "step": 86
+    },
+    {
+      "epoch": 0.009399562434162548,
+      "grad_norm": 7.3067827224731445,
+      "learning_rate": 9.068627601992598e-05,
+      "loss": 8.5148,
+      "step": 87
+    },
+    {
+      "epoch": 0.009507603381681657,
+      "grad_norm": 7.735067844390869,
+      "learning_rate": 9.045084971874738e-05,
+      "loss": 8.0955,
+      "step": 88
+    },
+    {
+      "epoch": 0.009615644329200767,
+      "grad_norm": 6.7928690910339355,
+      "learning_rate": 9.021279861989885e-05,
+      "loss": 7.7918,
+      "step": 89
+    },
+    {
+      "epoch": 0.009723685276719877,
+      "grad_norm": 7.215407371520996,
+      "learning_rate": 8.997213817017507e-05,
+      "loss": 8.0724,
+      "step": 90
+    },
+    {
+      "epoch": 0.009831726224238986,
+      "grad_norm": 6.715408802032471,
+      "learning_rate": 8.972888398568772e-05,
+      "loss": 7.9735,
+      "step": 91
+    },
+    {
+      "epoch": 0.009939767171758096,
+      "grad_norm": 6.818831443786621,
+      "learning_rate": 8.948305185085225e-05,
+      "loss": 8.0466,
+      "step": 92
+    },
+    {
+      "epoch": 0.010047808119277206,
+      "grad_norm": 7.25744104385376,
+      "learning_rate": 8.92346577173636e-05,
+      "loss": 8.3633,
+      "step": 93
+    },
+    {
+      "epoch": 0.010155849066796317,
+      "grad_norm": 7.1355180740356445,
+      "learning_rate": 8.898371770316111e-05,
+      "loss": 8.0414,
+      "step": 94
+    },
+    {
+      "epoch": 0.010263890014315425,
+      "grad_norm": 6.755433559417725,
+      "learning_rate": 8.873024809138272e-05,
+      "loss": 8.509,
+      "step": 95
+    },
+    {
+      "epoch": 0.010371930961834535,
+      "grad_norm": 7.057644367218018,
+      "learning_rate": 8.847426532930831e-05,
+      "loss": 7.8167,
+      "step": 96
+    },
+    {
+      "epoch": 0.010479971909353646,
+      "grad_norm": 7.3454508781433105,
+      "learning_rate": 8.821578602729242e-05,
+      "loss": 7.7997,
+      "step": 97
+    },
+    {
+      "epoch": 0.010588012856872754,
+      "grad_norm": 7.287400722503662,
+      "learning_rate": 8.795482695768658e-05,
+      "loss": 8.0839,
+      "step": 98
+    },
+    {
+      "epoch": 0.010696053804391864,
+      "grad_norm": 6.911624431610107,
+      "learning_rate": 8.769140505375085e-05,
+      "loss": 7.6225,
+      "step": 99
+    },
+    {
+      "epoch": 0.010804094751910975,
+      "grad_norm": 7.923533916473389,
+      "learning_rate": 8.742553740855506e-05,
+      "loss": 7.4759,
+      "step": 100
+    },
+    {
+      "epoch": 0.010804094751910975,
+      "eval_loss": 2.1499531269073486,
+      "eval_runtime": 115.0497,
+      "eval_samples_per_second": 135.498,
+      "eval_steps_per_second": 33.881,
+      "step": 100
+    },
+    {
+      "epoch": 0.010912135699430083,
+      "grad_norm": 11.213050842285156,
+      "learning_rate": 8.715724127386972e-05,
+      "loss": 9.7943,
+      "step": 101
+    },
+    {
+      "epoch": 0.011020176646949193,
+      "grad_norm": 11.380208969116211,
+      "learning_rate": 8.688653405904652e-05,
+      "loss": 9.5812,
+      "step": 102
+    },
+    {
+      "epoch": 0.011128217594468304,
+      "grad_norm": 9.7195405960083,
+      "learning_rate": 8.661343332988869e-05,
+      "loss": 9.6675,
+      "step": 103
+    },
+    {
+      "epoch": 0.011236258541987414,
+      "grad_norm": 7.468715190887451,
+      "learning_rate": 8.633795680751116e-05,
+      "loss": 9.341,
+      "step": 104
+    },
+    {
+      "epoch": 0.011344299489506523,
+      "grad_norm": 6.432350158691406,
+      "learning_rate": 8.606012236719073e-05,
+      "loss": 9.7432,
+      "step": 105
+    },
+    {
+      "epoch": 0.011452340437025633,
+      "grad_norm": 7.584165573120117,
+      "learning_rate": 8.577994803720606e-05,
+      "loss": 10.3697,
+      "step": 106
+    },
+    {
+      "epoch": 0.011560381384544743,
+      "grad_norm": 7.650453090667725,
+      "learning_rate": 8.549745199766792e-05,
+      "loss": 10.0901,
+      "step": 107
+    },
+    {
+      "epoch": 0.011668422332063852,
+      "grad_norm": 7.4186201095581055,
+      "learning_rate": 8.521265257933948e-05,
+      "loss": 9.5149,
+      "step": 108
+    },
+    {
+      "epoch": 0.011776463279582962,
+      "grad_norm": 7.989743709564209,
+      "learning_rate": 8.492556826244687e-05,
+      "loss": 10.4639,
+      "step": 109
+    },
+    {
+      "epoch": 0.011884504227102072,
+      "grad_norm": 8.052087783813477,
+      "learning_rate": 8.463621767547998e-05,
+      "loss": 10.1572,
+      "step": 110
+    },
+    {
+      "epoch": 0.01199254517462118,
+      "grad_norm": 8.013016700744629,
+      "learning_rate": 8.434461959398376e-05,
+      "loss": 10.0648,
+      "step": 111
+    },
+    {
+      "epoch": 0.012100586122140291,
+      "grad_norm": 7.510498046875,
+      "learning_rate": 8.405079293933986e-05,
+      "loss": 9.7153,
+      "step": 112
+    },
+    {
+      "epoch": 0.012208627069659401,
+      "grad_norm": 7.5188703536987305,
+      "learning_rate": 8.375475677753881e-05,
+      "loss": 9.5578,
+      "step": 113
+    },
+    {
+      "epoch": 0.012316668017178511,
+      "grad_norm": 7.69437837600708,
+      "learning_rate": 8.345653031794292e-05,
+      "loss": 10.7936,
+      "step": 114
+    },
+    {
+      "epoch": 0.01242470896469762,
+      "grad_norm": 8.049108505249023,
+      "learning_rate": 8.315613291203976e-05,
+      "loss": 10.2677,
+      "step": 115
+    },
+    {
+      "epoch": 0.01253274991221673,
+      "grad_norm": 8.205089569091797,
+      "learning_rate": 8.285358405218655e-05,
+      "loss": 9.8414,
+      "step": 116
+    },
+    {
+      "epoch": 0.01264079085973584,
+      "grad_norm": 8.370574951171875,
+      "learning_rate": 8.25489033703452e-05,
+      "loss": 8.659,
+      "step": 117
+    },
+    {
+      "epoch": 0.012748831807254949,
+      "grad_norm": 7.88584566116333,
+      "learning_rate": 8.224211063680853e-05,
+      "loss": 8.6691,
+      "step": 118
+    },
+    {
+      "epoch": 0.01285687275477406,
+      "grad_norm": 8.422807693481445,
+      "learning_rate": 8.19332257589174e-05,
+      "loss": 8.9626,
+      "step": 119
+    },
+    {
+      "epoch": 0.01296491370229317,
+      "grad_norm": 7.968113899230957,
+      "learning_rate": 8.162226877976887e-05,
+      "loss": 9.0886,
+      "step": 120
+    },
+    {
+      "epoch": 0.013072954649812278,
+      "grad_norm": 7.314736843109131,
+      "learning_rate": 8.130925987691569e-05,
+      "loss": 8.2871,
+      "step": 121
+    },
+    {
+      "epoch": 0.013180995597331388,
+      "grad_norm": 6.700660228729248,
+      "learning_rate": 8.099421936105702e-05,
+      "loss": 8.2856,
+      "step": 122
+    },
+    {
+      "epoch": 0.013289036544850499,
+      "grad_norm": 6.392932415008545,
+      "learning_rate": 8.067716767472045e-05,
+      "loss": 8.2655,
+      "step": 123
+    },
+    {
+      "epoch": 0.013397077492369609,
+      "grad_norm": 6.0710577964782715,
+      "learning_rate": 8.035812539093557e-05,
+      "loss": 8.4146,
+      "step": 124
+    },
+    {
+      "epoch": 0.013505118439888717,
+      "grad_norm": 6.290673732757568,
+      "learning_rate": 8.003711321189895e-05,
+      "loss": 8.4104,
+      "step": 125
+    },
+    {
+      "epoch": 0.013613159387407828,
+      "grad_norm": 5.992330074310303,
+      "learning_rate": 7.971415196763088e-05,
+      "loss": 8.608,
+      "step": 126
+    },
+    {
+      "epoch": 0.013721200334926938,
+      "grad_norm": 6.178021430969238,
+      "learning_rate": 7.938926261462366e-05,
+      "loss": 7.76,
+      "step": 127
+    },
+    {
+      "epoch": 0.013829241282446047,
+      "grad_norm": 7.223654747009277,
+      "learning_rate": 7.906246623448183e-05,
+      "loss": 8.2067,
+      "step": 128
+    },
+    {
+      "epoch": 0.013937282229965157,
+      "grad_norm": 6.30562686920166,
+      "learning_rate": 7.873378403255419e-05,
+      "loss": 8.2667,
+      "step": 129
+    },
+    {
+      "epoch": 0.014045323177484267,
+      "grad_norm": 6.786837100982666,
+      "learning_rate": 7.840323733655778e-05,
+      "loss": 8.6115,
+      "step": 130
+    },
+    {
+      "epoch": 0.014153364125003376,
+      "grad_norm": 6.058274745941162,
+      "learning_rate": 7.807084759519405e-05,
+      "loss": 8.105,
+      "step": 131
+    },
+    {
+      "epoch": 0.014261405072522486,
+      "grad_norm": 6.321048259735107,
+      "learning_rate": 7.773663637675694e-05,
+      "loss": 8.023,
+      "step": 132
+    },
+    {
+      "epoch": 0.014369446020041596,
+      "grad_norm": 6.666811943054199,
+      "learning_rate": 7.740062536773352e-05,
+      "loss": 8.4926,
+      "step": 133
+    },
+    {
+      "epoch": 0.014477486967560705,
+      "grad_norm": 5.980512619018555,
+      "learning_rate": 7.706283637139658e-05,
+      "loss": 8.2539,
+      "step": 134
+    },
+    {
+      "epoch": 0.014585527915079815,
+      "grad_norm": 6.472214698791504,
+      "learning_rate": 7.672329130639005e-05,
+      "loss": 7.9661,
+      "step": 135
+    },
+    {
+      "epoch": 0.014693568862598925,
+      "grad_norm": 6.751770973205566,
+      "learning_rate": 7.638201220530665e-05,
+      "loss": 8.0303,
+      "step": 136
+    },
+    {
+      "epoch": 0.014801609810118035,
+      "grad_norm": 6.003720283508301,
+      "learning_rate": 7.603902121325813e-05,
+      "loss": 7.6914,
+      "step": 137
+    },
+    {
+      "epoch": 0.014909650757637144,
+      "grad_norm": 6.814517021179199,
+      "learning_rate": 7.569434058643844e-05,
+      "loss": 8.0067,
+      "step": 138
+    },
+    {
+      "epoch": 0.015017691705156254,
+      "grad_norm": 6.680156230926514,
+      "learning_rate": 7.534799269067953e-05,
+      "loss": 7.8388,
+      "step": 139
+    },
+    {
+      "epoch": 0.015125732652675365,
+      "grad_norm": 6.444919109344482,
+      "learning_rate": 7.500000000000001e-05,
+      "loss": 7.9921,
+      "step": 140
+    },
+    {
+      "epoch": 0.015233773600194473,
+      "grad_norm": 6.862926959991455,
+      "learning_rate": 7.465038509514688e-05,
+      "loss": 8.0343,
+      "step": 141
+    },
+    {
+      "epoch": 0.015341814547713583,
+      "grad_norm": 6.10462760925293,
+      "learning_rate": 7.42991706621303e-05,
+      "loss": 7.2419,
+      "step": 142
+    },
+    {
+      "epoch": 0.015449855495232694,
+      "grad_norm": 7.222691059112549,
+      "learning_rate": 7.394637949075154e-05,
+      "loss": 7.7565,
+      "step": 143
+    },
+    {
+      "epoch": 0.015557896442751802,
+      "grad_norm": 6.676332473754883,
+      "learning_rate": 7.35920344731241e-05,
+      "loss": 7.5405,
+      "step": 144
+    },
+    {
+      "epoch": 0.015665937390270914,
+      "grad_norm": 7.371861934661865,
+      "learning_rate": 7.323615860218843e-05,
+      "loss": 7.6452,
+      "step": 145
+    },
+    {
+      "epoch": 0.015773978337790023,
+      "grad_norm": 7.728967189788818,
+      "learning_rate": 7.287877497021978e-05,
+      "loss": 8.5353,
+      "step": 146
+    },
+    {
+      "epoch": 0.01588201928530913,
+      "grad_norm": 6.466920852661133,
+      "learning_rate": 7.251990676732984e-05,
+      "loss": 6.6836,
+      "step": 147
+    },
+    {
+      "epoch": 0.015990060232828243,
+      "grad_norm": 7.410046577453613,
+      "learning_rate": 7.215957727996207e-05,
+      "loss": 7.6081,
+      "step": 148
+    },
+    {
+      "epoch": 0.01609810118034735,
+      "grad_norm": 6.660158157348633,
+      "learning_rate": 7.179780988938051e-05,
+      "loss": 6.6185,
+      "step": 149
+    },
+    {
+      "epoch": 0.01620614212786646,
+      "grad_norm": 7.526998996734619,
+      "learning_rate": 7.143462807015271e-05,
+      "loss": 7.826,
+      "step": 150
+    },
+    {
+      "epoch": 0.01620614212786646,
+      "eval_loss": 2.116058349609375,
+      "eval_runtime": 115.2532,
+      "eval_samples_per_second": 135.259,
+      "eval_steps_per_second": 33.821,
+      "step": 150
+    },
+    {
+      "epoch": 0.016314183075385572,
+      "grad_norm": 8.569231986999512,
+      "learning_rate": 7.107005538862646e-05,
+      "loss": 9.2097,
+      "step": 151
+    },
+    {
+      "epoch": 0.01642222402290468,
+      "grad_norm": 7.914257526397705,
+      "learning_rate": 7.07041155014006e-05,
+      "loss": 9.6914,
+      "step": 152
+    },
+    {
+      "epoch": 0.01653026497042379,
+      "grad_norm": 6.4140496253967285,
+      "learning_rate": 7.033683215379002e-05,
+      "loss": 9.0563,
+      "step": 153
+    },
+    {
+      "epoch": 0.0166383059179429,
+      "grad_norm": 6.3061981201171875,
+      "learning_rate": 6.996822917828477e-05,
+      "loss": 9.6594,
+      "step": 154
+    },
+    {
+      "epoch": 0.01674634686546201,
+      "grad_norm": 6.62816047668457,
+      "learning_rate": 6.959833049300377e-05,
+      "loss": 9.9273,
+      "step": 155
+    },
+    {
+      "epoch": 0.01685438781298112,
+      "grad_norm": 6.893472194671631,
+      "learning_rate": 6.922716010014255e-05,
+      "loss": 10.0839,
+      "step": 156
+    },
+    {
+      "epoch": 0.01696242876050023,
+      "grad_norm": 7.660455226898193,
+      "learning_rate": 6.885474208441603e-05,
+      "loss": 9.7825,
+      "step": 157
+    },
+    {
+      "epoch": 0.01707046970801934,
+      "grad_norm": 7.464423656463623,
+      "learning_rate": 6.848110061149556e-05,
+      "loss": 9.9369,
+      "step": 158
+    },
+    {
+      "epoch": 0.017178510655538447,
+      "grad_norm": 7.637061595916748,
+      "learning_rate": 6.810625992644085e-05,
+      "loss": 10.5188,
+      "step": 159
+    },
+    {
+      "epoch": 0.01728655160305756,
+      "grad_norm": 7.390578269958496,
+      "learning_rate": 6.773024435212678e-05,
+      "loss": 9.4138,
+      "step": 160
+    },
+    {
+      "epoch": 0.017394592550576668,
+      "grad_norm": 8.361207008361816,
+      "learning_rate": 6.735307828766515e-05,
+      "loss": 10.1234,
+      "step": 161
+    },
+    {
+      "epoch": 0.01750263349809578,
+      "grad_norm": 7.585420608520508,
+      "learning_rate": 6.697478620682137e-05,
+      "loss": 9.8918,
+      "step": 162
+    },
+    {
+      "epoch": 0.01761067444561489,
+      "grad_norm": 8.020647048950195,
+      "learning_rate": 6.659539265642643e-05,
+      "loss": 10.073,
+      "step": 163
+    },
+    {
+      "epoch": 0.017718715393133997,
+      "grad_norm": 8.187236785888672,
+      "learning_rate": 6.621492225478414e-05,
+      "loss": 10.2494,
+      "step": 164
+    },
+    {
+      "epoch": 0.01782675634065311,
+      "grad_norm": 7.50603723526001,
+      "learning_rate": 6.583339969007363e-05,
+      "loss": 9.1249,
+      "step": 165
+    },
+    {
+      "epoch": 0.017934797288172218,
+      "grad_norm": 7.881903648376465,
+      "learning_rate": 6.545084971874738e-05,
+      "loss": 10.1012,
+      "step": 166
+    },
+    {
+      "epoch": 0.018042838235691326,
+      "grad_norm": 9.54574966430664,
+      "learning_rate": 6.506729716392481e-05,
+      "loss": 9.4214,
+      "step": 167
+    },
+    {
+      "epoch": 0.018150879183210438,
+      "grad_norm": 7.5880255699157715,
+      "learning_rate": 6.468276691378155e-05,
+      "loss": 8.8946,
+      "step": 168
+    },
+    {
+      "epoch": 0.018258920130729547,
+      "grad_norm": 7.4278035163879395,
+      "learning_rate": 6.429728391993446e-05,
+      "loss": 8.6887,
+      "step": 169
+    },
+    {
+      "epoch": 0.018366961078248655,
+      "grad_norm": 6.724630832672119,
+      "learning_rate": 6.391087319582264e-05,
+      "loss": 8.5969,
+      "step": 170
+    },
+    {
+      "epoch": 0.018475002025767767,
+      "grad_norm": 6.967141628265381,
+      "learning_rate": 6.35235598150842e-05,
+      "loss": 9.0554,
+      "step": 171
+    },
+    {
+      "epoch": 0.018583042973286876,
+      "grad_norm": 6.539910793304443,
+      "learning_rate": 6.313536890992935e-05,
+      "loss": 8.622,
+      "step": 172
+    },
+    {
+      "epoch": 0.018691083920805984,
+      "grad_norm": 7.247252464294434,
+      "learning_rate": 6.274632566950967e-05,
+      "loss": 9.3202,
+      "step": 173
+    },
+    {
+      "epoch": 0.018799124868325096,
+      "grad_norm": 6.7946319580078125,
+      "learning_rate": 6.235645533828349e-05,
+      "loss": 8.0408,
+      "step": 174
+    },
+    {
+      "epoch": 0.018907165815844205,
+      "grad_norm": 7.553107738494873,
+      "learning_rate": 6.19657832143779e-05,
+      "loss": 8.627,
+      "step": 175
+    },
+    {
+      "epoch": 0.019015206763363313,
+      "grad_norm": 6.53674840927124,
+      "learning_rate": 6.157433464794716e-05,
+      "loss": 8.1034,
+      "step": 176
+    },
+    {
+      "epoch": 0.019123247710882425,
+      "grad_norm": 6.148805618286133,
+      "learning_rate": 6.118213503952779e-05,
+      "loss": 7.715,
+      "step": 177
+    },
+    {
+      "epoch": 0.019231288658401534,
+      "grad_norm": 6.221681594848633,
+      "learning_rate": 6.078920983839031e-05,
+      "loss": 8.1824,
+      "step": 178
+    },
+    {
+      "epoch": 0.019339329605920642,
+      "grad_norm": 5.551029682159424,
+      "learning_rate": 6.0395584540887963e-05,
+      "loss": 7.275,
+      "step": 179
+    },
+    {
+      "epoch": 0.019447370553439754,
+      "grad_norm": 5.740414619445801,
+      "learning_rate": 6.0001284688802226e-05,
+      "loss": 7.5641,
+      "step": 180
+    },
+    {
+      "epoch": 0.019555411500958863,
+      "grad_norm": 6.624487400054932,
+      "learning_rate": 5.960633586768543e-05,
+      "loss": 7.5697,
+      "step": 181
+    },
+    {
+      "epoch": 0.01966345244847797,
+      "grad_norm": 7.313840389251709,
+      "learning_rate": 5.921076370520058e-05,
+      "loss": 8.4292,
+      "step": 182
+    },
+    {
+      "epoch": 0.019771493395997083,
+      "grad_norm": 5.5785675048828125,
+      "learning_rate": 5.8814593869458455e-05,
+      "loss": 7.3243,
+      "step": 183
+    },
+    {
+      "epoch": 0.019879534343516192,
+      "grad_norm": 7.2685675621032715,
+      "learning_rate": 5.841785206735192e-05,
+      "loss": 9.1149,
+      "step": 184
+    },
+    {
+      "epoch": 0.019987575291035304,
+      "grad_norm": 6.239570617675781,
+      "learning_rate": 5.8020564042888015e-05,
+      "loss": 8.3288,
+      "step": 185
+    },
+    {
+      "epoch": 0.020095616238554413,
+      "grad_norm": 5.983908176422119,
+      "learning_rate": 5.762275557551727e-05,
+      "loss": 8.2129,
+      "step": 186
+    },
+    {
+      "epoch": 0.02020365718607352,
+      "grad_norm": 6.797692775726318,
+      "learning_rate": 5.7224452478461064e-05,
+      "loss": 8.0456,
+      "step": 187
+    },
+    {
+      "epoch": 0.020311698133592633,
+      "grad_norm": 6.501678466796875,
+      "learning_rate": 5.682568059703659e-05,
+      "loss": 8.5035,
+      "step": 188
+    },
+    {
+      "epoch": 0.02041973908111174,
+      "grad_norm": 6.174941539764404,
+      "learning_rate": 5.642646580697973e-05,
+      "loss": 7.9314,
+      "step": 189
+    },
+    {
+      "epoch": 0.02052778002863085,
+      "grad_norm": 7.605291843414307,
+      "learning_rate": 5.602683401276615e-05,
+      "loss": 8.7727,
+      "step": 190
+    },
+    {
+      "epoch": 0.020635820976149962,
+      "grad_norm": 6.667246341705322,
+      "learning_rate": 5.562681114593028e-05,
+      "loss": 7.9204,
+      "step": 191
+    },
+    {
+      "epoch": 0.02074386192366907,
+      "grad_norm": 6.500163555145264,
+      "learning_rate": 5.522642316338268e-05,
+      "loss": 7.6879,
+      "step": 192
+    },
+    {
+      "epoch": 0.02085190287118818,
+      "grad_norm": 6.8648858070373535,
+      "learning_rate": 5.482569604572576e-05,
+      "loss": 8.0821,
+      "step": 193
+    },
+    {
+      "epoch": 0.02095994381870729,
+      "grad_norm": 6.480758190155029,
+      "learning_rate": 5.442465579556793e-05,
+      "loss": 7.4802,
+      "step": 194
+    },
+    {
+      "epoch": 0.0210679847662264,
+      "grad_norm": 6.604392051696777,
+      "learning_rate": 5.402332843583631e-05,
+      "loss": 7.7247,
+      "step": 195
+    },
+    {
+      "epoch": 0.021176025713745508,
+      "grad_norm": 6.822199821472168,
+      "learning_rate": 5.3621740008088126e-05,
+      "loss": 7.599,
+      "step": 196
+    },
+    {
+      "epoch": 0.02128406666126462,
+      "grad_norm": 6.770035743713379,
+      "learning_rate": 5.321991657082097e-05,
+      "loss": 8.1888,
+      "step": 197
+    },
+    {
+      "epoch": 0.02139210760878373,
+      "grad_norm": 6.449174404144287,
+      "learning_rate": 5.281788419778187e-05,
+      "loss": 7.8245,
+      "step": 198
+    },
+    {
+      "epoch": 0.021500148556302837,
+      "grad_norm": 6.712276935577393,
+      "learning_rate": 5.2415668976275355e-05,
+      "loss": 7.6638,
+      "step": 199
+    },
+    {
+      "epoch": 0.02160818950382195,
+      "grad_norm": 7.5627593994140625,
+      "learning_rate": 5.201329700547076e-05,
+      "loss": 7.5391,
+      "step": 200
+    },
+    {
+      "epoch": 0.02160818950382195,
+      "eval_loss": 2.137317419052124,
+      "eval_runtime": 115.1004,
+      "eval_samples_per_second": 135.438,
+      "eval_steps_per_second": 33.866,
+      "step": 200
+    },
+    {
+      "epoch": 0.021716230451341058,
+      "grad_norm": 11.5592622756958,
+      "learning_rate": 5.161079439470866e-05,
+      "loss": 9.7146,
+      "step": 201
+    },
+    {
+      "epoch": 0.021824271398860166,
+      "grad_norm": 11.015775680541992,
+      "learning_rate": 5.1208187261806615e-05,
+      "loss": 10.3395,
+      "step": 202
+    },
+    {
+      "epoch": 0.02193231234637928,
+      "grad_norm": 10.193652153015137,
+      "learning_rate": 5.080550173136457e-05,
+      "loss": 9.9166,
+      "step": 203
+    },
+    {
+      "epoch": 0.022040353293898387,
+      "grad_norm": 9.768710136413574,
+      "learning_rate": 5.0402763933069496e-05,
+      "loss": 10.5806,
+      "step": 204
+    },
+    {
+      "epoch": 0.0221483942414175,
+      "grad_norm": 9.029803276062012,
+      "learning_rate": 5e-05,
+      "loss": 10.5787,
+      "step": 205
+    },
+    {
+      "epoch": 0.022256435188936607,
+      "grad_norm": 7.742230415344238,
+      "learning_rate": 4.9597236066930516e-05,
+      "loss": 9.6428,
+      "step": 206
+    },
+    {
+      "epoch": 0.022364476136455716,
+      "grad_norm": 6.682606220245361,
+      "learning_rate": 4.919449826863544e-05,
+      "loss": 9.7322,
+      "step": 207
+    },
+    {
+      "epoch": 0.022472517083974828,
+      "grad_norm": 6.897256851196289,
+      "learning_rate": 4.87918127381934e-05,
+      "loss": 10.0854,
+      "step": 208
+    },
+    {
+      "epoch": 0.022580558031493937,
+      "grad_norm": 7.059166431427002,
+      "learning_rate": 4.8389205605291365e-05,
+      "loss": 10.4431,
+      "step": 209
+    },
+    {
+      "epoch": 0.022688598979013045,
+      "grad_norm": 7.340524673461914,
+      "learning_rate": 4.798670299452926e-05,
+      "loss": 8.8217,
+      "step": 210
+    },
+    {
+      "epoch": 0.022796639926532157,
+      "grad_norm": 7.51647424697876,
+      "learning_rate": 4.758433102372466e-05,
+      "loss": 9.5277,
+      "step": 211
+    },
+    {
+      "epoch": 0.022904680874051266,
+      "grad_norm": 8.075295448303223,
+      "learning_rate": 4.7182115802218126e-05,
+      "loss": 10.0584,
+      "step": 212
+    },
+    {
+      "epoch": 0.023012721821570374,
+      "grad_norm": 7.203787803649902,
+      "learning_rate": 4.678008342917903e-05,
+      "loss": 9.9091,
+      "step": 213
+    },
+    {
+      "epoch": 0.023120762769089486,
+      "grad_norm": 7.923792362213135,
+      "learning_rate": 4.6378259991911886e-05,
+      "loss": 9.0695,
+      "step": 214
+    },
+    {
+      "epoch": 0.023228803716608595,
+      "grad_norm": 7.329343318939209,
+      "learning_rate": 4.597667156416371e-05,
+      "loss": 9.3932,
+      "step": 215
+    },
+    {
+      "epoch": 0.023336844664127703,
+      "grad_norm": 6.979767799377441,
+      "learning_rate": 4.5575344204432084e-05,
+      "loss": 8.203,
+      "step": 216
+    },
+    {
+      "epoch": 0.023444885611646815,
+      "grad_norm": 7.324328422546387,
+      "learning_rate": 4.5174303954274244e-05,
+      "loss": 8.6626,
+      "step": 217
+    },
+    {
+      "epoch": 0.023552926559165924,
+      "grad_norm": 7.1822710037231445,
+      "learning_rate": 4.477357683661734e-05,
+      "loss": 7.6541,
+      "step": 218
+    },
+    {
+      "epoch": 0.023660967506685032,
+      "grad_norm": 8.961649894714355,
+      "learning_rate": 4.437318885406973e-05,
+      "loss": 9.2032,
+      "step": 219
+    },
+    {
+      "epoch": 0.023769008454204144,
+      "grad_norm": 7.797355651855469,
+      "learning_rate": 4.397316598723385e-05,
+      "loss": 9.0441,
+      "step": 220
+    },
+    {
+      "epoch": 0.023877049401723253,
+      "grad_norm": 7.194826602935791,
+      "learning_rate": 4.3573534193020274e-05,
+      "loss": 8.3579,
+      "step": 221
+    },
+    {
+      "epoch": 0.02398509034924236,
+      "grad_norm": 6.243997097015381,
+      "learning_rate": 4.317431940296343e-05,
+      "loss": 7.7125,
+      "step": 222
+    },
+    {
+      "epoch": 0.024093131296761473,
+      "grad_norm": 5.78803825378418,
+      "learning_rate": 4.277554752153895e-05,
+      "loss": 8.0223,
+      "step": 223
+    },
+    {
+      "epoch": 0.024201172244280582,
+      "grad_norm": 5.73882532119751,
+      "learning_rate": 4.237724442448273e-05,
+      "loss": 7.8321,
+      "step": 224
+    },
+    {
+      "epoch": 0.02430921319179969,
+      "grad_norm": 6.035843372344971,
+      "learning_rate": 4.197943595711198e-05,
+      "loss": 8.6849,
+      "step": 225
+    },
+    {
+      "epoch": 0.024417254139318802,
+      "grad_norm": 6.8568949699401855,
+      "learning_rate": 4.1582147932648074e-05,
+      "loss": 9.0212,
+      "step": 226
+    },
+    {
+      "epoch": 0.02452529508683791,
+      "grad_norm": 6.396918296813965,
+      "learning_rate": 4.118540613054156e-05,
+      "loss": 7.9765,
+      "step": 227
+    },
+    {
+      "epoch": 0.024633336034357023,
+      "grad_norm": 5.814195156097412,
+      "learning_rate": 4.078923629479943e-05,
+      "loss": 7.8777,
+      "step": 228
+    },
+    {
+      "epoch": 0.02474137698187613,
+      "grad_norm": 6.143547058105469,
+      "learning_rate": 4.039366413231458e-05,
+      "loss": 7.6745,
+      "step": 229
+    },
+    {
+      "epoch": 0.02484941792939524,
+      "grad_norm": 5.96578311920166,
+      "learning_rate": 3.9998715311197785e-05,
+      "loss": 8.0483,
+      "step": 230
+    },
+    {
+      "epoch": 0.024957458876914352,
+      "grad_norm": 6.287611484527588,
+      "learning_rate": 3.960441545911204e-05,
+      "loss": 8.4885,
+      "step": 231
+    },
+    {
+      "epoch": 0.02506549982443346,
+      "grad_norm": 5.993619918823242,
+      "learning_rate": 3.92107901616097e-05,
+      "loss": 8.3242,
+      "step": 232
+    },
+    {
+      "epoch": 0.02517354077195257,
+      "grad_norm": 6.037485599517822,
+      "learning_rate": 3.8817864960472236e-05,
+      "loss": 8.4465,
+      "step": 233
+    },
+    {
+      "epoch": 0.02528158171947168,
+      "grad_norm": 5.9138503074646,
+      "learning_rate": 3.842566535205286e-05,
+      "loss": 7.7514,
+      "step": 234
+    },
+    {
+      "epoch": 0.02538962266699079,
+      "grad_norm": 6.0712738037109375,
+      "learning_rate": 3.803421678562213e-05,
+      "loss": 7.8649,
+      "step": 235
+    },
+    {
+      "epoch": 0.025497663614509898,
+      "grad_norm": 6.2299652099609375,
+      "learning_rate": 3.764354466171652e-05,
+      "loss": 8.2885,
+      "step": 236
+    },
+    {
+      "epoch": 0.02560570456202901,
+      "grad_norm": 6.322656154632568,
+      "learning_rate": 3.725367433049033e-05,
+      "loss": 7.0295,
+      "step": 237
+    },
+    {
+      "epoch": 0.02571374550954812,
+      "grad_norm": 5.935736656188965,
+      "learning_rate": 3.6864631090070655e-05,
+      "loss": 7.6595,
+      "step": 238
+    },
+    {
+      "epoch": 0.025821786457067227,
+      "grad_norm": 7.882625102996826,
+      "learning_rate": 3.6476440184915815e-05,
+      "loss": 8.8297,
+      "step": 239
+    },
+    {
+      "epoch": 0.02592982740458634,
+      "grad_norm": 6.351822376251221,
+      "learning_rate": 3.608912680417737e-05,
+      "loss": 7.6028,
+      "step": 240
+    },
+    {
+      "epoch": 0.026037868352105448,
+      "grad_norm": 6.599503517150879,
+      "learning_rate": 3.570271608006555e-05,
+      "loss": 7.4339,
+      "step": 241
+    },
+    {
+      "epoch": 0.026145909299624556,
+      "grad_norm": 6.108438491821289,
+      "learning_rate": 3.531723308621847e-05,
+      "loss": 7.5243,
+      "step": 242
+    },
+    {
+      "epoch": 0.026253950247143668,
+      "grad_norm": 6.1683783531188965,
+      "learning_rate": 3.493270283607522e-05,
+      "loss": 7.6404,
+      "step": 243
+    },
+    {
+      "epoch": 0.026361991194662777,
+      "grad_norm": 6.002394676208496,
+      "learning_rate": 3.4549150281252636e-05,
+      "loss": 7.7044,
+      "step": 244
+    },
+    {
+      "epoch": 0.026470032142181885,
+      "grad_norm": 6.1193156242370605,
+      "learning_rate": 3.4166600309926387e-05,
+      "loss": 7.3726,
+      "step": 245
+    },
+    {
+      "epoch": 0.026578073089700997,
+      "grad_norm": 5.775542259216309,
+      "learning_rate": 3.3785077745215873e-05,
+      "loss": 7.6895,
+      "step": 246
+    },
+    {
+      "epoch": 0.026686114037220106,
+      "grad_norm": 6.05790376663208,
+      "learning_rate": 3.340460734357359e-05,
+      "loss": 7.3421,
+      "step": 247
+    },
+    {
+      "epoch": 0.026794154984739218,
+      "grad_norm": 6.469996929168701,
+      "learning_rate": 3.3025213793178646e-05,
+      "loss": 7.5728,
+      "step": 248
+    },
+    {
+      "epoch": 0.026902195932258326,
+      "grad_norm": 6.183184623718262,
+      "learning_rate": 3.264692171233485e-05,
+      "loss": 7.4667,
+      "step": 249
+    },
+    {
+      "epoch": 0.027010236879777435,
+      "grad_norm": 7.3603692054748535,
+      "learning_rate": 3.226975564787322e-05,
+      "loss": 7.5569,
+      "step": 250
+    },
+    {
+      "epoch": 0.027010236879777435,
+      "eval_loss": 2.0691401958465576,
+      "eval_runtime": 115.3475,
+      "eval_samples_per_second": 135.148,
+      "eval_steps_per_second": 33.794,
+      "step": 250
+    },
+    {
+      "epoch": 0.027118277827296547,
+      "grad_norm": 9.313230514526367,
+      "learning_rate": 3.189374007355917e-05,
+      "loss": 9.5615,
+      "step": 251
+    },
+    {
+      "epoch": 0.027226318774815655,
+      "grad_norm": 7.8498969078063965,
+      "learning_rate": 3.151889938850445e-05,
+      "loss": 9.9529,
+      "step": 252
+    },
+    {
+      "epoch": 0.027334359722334764,
+      "grad_norm": 7.026669025421143,
+      "learning_rate": 3.114525791558398e-05,
+      "loss": 9.4969,
+      "step": 253
+    },
+    {
+      "epoch": 0.027442400669853876,
+      "grad_norm": 6.743854522705078,
+      "learning_rate": 3.0772839899857464e-05,
+      "loss": 9.6743,
+      "step": 254
+    },
+    {
+      "epoch": 0.027550441617372984,
+      "grad_norm": 6.488364219665527,
+      "learning_rate": 3.0401669506996256e-05,
+      "loss": 10.3293,
+      "step": 255
+    },
+    {
+      "epoch": 0.027658482564892093,
+      "grad_norm": 6.20983362197876,
+      "learning_rate": 3.003177082171523e-05,
+      "loss": 9.623,
+      "step": 256
+    },
+    {
+      "epoch": 0.027766523512411205,
+      "grad_norm": 6.174509525299072,
+      "learning_rate": 2.9663167846209998e-05,
+      "loss": 9.7071,
+      "step": 257
+    },
+    {
+      "epoch": 0.027874564459930314,
+      "grad_norm": 6.855802059173584,
+      "learning_rate": 2.9295884498599414e-05,
+      "loss": 9.6577,
+      "step": 258
+    },
+    {
+      "epoch": 0.027982605407449422,
+      "grad_norm": 6.102505683898926,
+      "learning_rate": 2.8929944611373554e-05,
+      "loss": 9.0591,
+      "step": 259
+    },
+    {
+      "epoch": 0.028090646354968534,
+      "grad_norm": 6.577708721160889,
+      "learning_rate": 2.8565371929847284e-05,
+      "loss": 9.607,
+      "step": 260
+    },
+    {
+      "epoch": 0.028198687302487643,
+      "grad_norm": 6.475739479064941,
+      "learning_rate": 2.8202190110619493e-05,
+      "loss": 9.1624,
+      "step": 261
+    },
+    {
+      "epoch": 0.02830672825000675,
+      "grad_norm": 7.7596001625061035,
+      "learning_rate": 2.784042272003794e-05,
+      "loss": 9.9397,
+      "step": 262
+    },
+    {
+      "epoch": 0.028414769197525863,
+      "grad_norm": 7.516853332519531,
+      "learning_rate": 2.7480093232670158e-05,
+      "loss": 9.8481,
+      "step": 263
+    },
+    {
+      "epoch": 0.02852281014504497,
+      "grad_norm": 7.379636287689209,
+      "learning_rate": 2.712122502978024e-05,
+      "loss": 10.2926,
+      "step": 264
+    },
+    {
+      "epoch": 0.02863085109256408,
+      "grad_norm": 8.061079025268555,
+      "learning_rate": 2.6763841397811573e-05,
+      "loss": 9.2398,
+      "step": 265
+    },
+    {
+      "epoch": 0.028738892040083192,
+      "grad_norm": 8.05746841430664,
+      "learning_rate": 2.64079655268759e-05,
+      "loss": 8.8153,
+      "step": 266
+    },
+    {
+      "epoch": 0.0288469329876023,
+      "grad_norm": 6.836451053619385,
+      "learning_rate": 2.605362050924848e-05,
+      "loss": 9.0268,
+      "step": 267
+    },
+    {
+      "epoch": 0.02895497393512141,
+      "grad_norm": 6.054388046264648,
+      "learning_rate": 2.57008293378697e-05,
+      "loss": 7.8709,
+      "step": 268
+    },
+    {
+      "epoch": 0.02906301488264052,
+      "grad_norm": 6.513256549835205,
+      "learning_rate": 2.534961490485313e-05,
+      "loss": 8.5975,
+      "step": 269
+    },
+    {
+      "epoch": 0.02917105583015963,
+      "grad_norm": 6.061592102050781,
+      "learning_rate": 2.500000000000001e-05,
+      "loss": 8.5225,
+      "step": 270
+    },
+    {
+      "epoch": 0.029279096777678742,
+      "grad_norm": 6.279740333557129,
+      "learning_rate": 2.4652007309320498e-05,
+      "loss": 8.2566,
+      "step": 271
+    },
+    {
+      "epoch": 0.02938713772519785,
+      "grad_norm": 7.238640785217285,
+      "learning_rate": 2.430565941356157e-05,
+      "loss": 8.6942,
+      "step": 272
+    },
+    {
+      "epoch": 0.02949517867271696,
+      "grad_norm": 5.981517314910889,
+      "learning_rate": 2.3960978786741877e-05,
+      "loss": 8.282,
+      "step": 273
+    },
+    {
+      "epoch": 0.02960321962023607,
+      "grad_norm": 5.895859718322754,
+      "learning_rate": 2.361798779469336e-05,
+      "loss": 7.5905,
+      "step": 274
+    },
+    {
+      "epoch": 0.02971126056775518,
+      "grad_norm": 5.94305944442749,
+      "learning_rate": 2.3276708693609943e-05,
+      "loss": 8.4324,
+      "step": 275
+    },
+    {
+      "epoch": 0.029819301515274288,
+      "grad_norm": 6.111569881439209,
+      "learning_rate": 2.2937163628603435e-05,
+      "loss": 8.4359,
+      "step": 276
+    },
+    {
+      "epoch": 0.0299273424627934,
+      "grad_norm": 6.170668601989746,
+      "learning_rate": 2.259937463226651e-05,
+      "loss": 8.044,
+      "step": 277
+    },
+    {
+      "epoch": 0.03003538341031251,
+      "grad_norm": 6.02022647857666,
+      "learning_rate": 2.2263363623243054e-05,
+      "loss": 7.4763,
+      "step": 278
+    },
+    {
+      "epoch": 0.030143424357831617,
+      "grad_norm": 6.0727643966674805,
+      "learning_rate": 2.192915240480596e-05,
+      "loss": 8.1001,
+      "step": 279
+    },
+    {
+      "epoch": 0.03025146530535073,
+      "grad_norm": 6.28542947769165,
+      "learning_rate": 2.1596762663442218e-05,
+      "loss": 8.3284,
+      "step": 280
+    },
+    {
+      "epoch": 0.030359506252869838,
+      "grad_norm": 5.896113872528076,
+      "learning_rate": 2.1266215967445824e-05,
+      "loss": 7.6581,
+      "step": 281
+    },
+    {
+      "epoch": 0.030467547200388946,
+      "grad_norm": 6.308555603027344,
+      "learning_rate": 2.0937533765518187e-05,
+      "loss": 8.4085,
+      "step": 282
+    },
+    {
+      "epoch": 0.030575588147908058,
+      "grad_norm": 6.037327289581299,
+      "learning_rate": 2.061073738537635e-05,
+      "loss": 7.9208,
+      "step": 283
+    },
+    {
+      "epoch": 0.030683629095427167,
+      "grad_norm": 5.352879524230957,
+      "learning_rate": 2.0285848032369137e-05,
+      "loss": 7.9334,
+      "step": 284
+    },
+    {
+      "epoch": 0.030791670042946275,
+      "grad_norm": 5.419848442077637,
+      "learning_rate": 1.996288678810105e-05,
+      "loss": 7.8057,
+      "step": 285
+    },
+    {
+      "epoch": 0.030899710990465387,
+      "grad_norm": 6.514065265655518,
+      "learning_rate": 1.9641874609064443e-05,
+      "loss": 8.2645,
+      "step": 286
+    },
+    {
+      "epoch": 0.031007751937984496,
+      "grad_norm": 5.476831912994385,
+      "learning_rate": 1.932283232527956e-05,
+      "loss": 7.5666,
+      "step": 287
+    },
+    {
+      "epoch": 0.031115792885503604,
+      "grad_norm": 6.499022960662842,
+      "learning_rate": 1.9005780638942982e-05,
+      "loss": 7.3987,
+      "step": 288
+    },
+    {
+      "epoch": 0.031223833833022716,
+      "grad_norm": 5.979431629180908,
+      "learning_rate": 1.8690740123084316e-05,
+      "loss": 8.1492,
+      "step": 289
+    },
+    {
+      "epoch": 0.03133187478054183,
+      "grad_norm": 6.137941837310791,
+      "learning_rate": 1.837773122023114e-05,
+      "loss": 7.8641,
+      "step": 290
+    },
+    {
+      "epoch": 0.03143991572806094,
+      "grad_norm": 5.927407741546631,
+      "learning_rate": 1.8066774241082612e-05,
+      "loss": 7.3963,
+      "step": 291
+    },
+    {
+      "epoch": 0.031547956675580045,
+      "grad_norm": 6.00791072845459,
+      "learning_rate": 1.7757889363191483e-05,
+      "loss": 7.7847,
+      "step": 292
+    },
+    {
+      "epoch": 0.031655997623099154,
+      "grad_norm": 6.854775905609131,
+      "learning_rate": 1.745109662965481e-05,
+      "loss": 8.8422,
+      "step": 293
+    },
+    {
+      "epoch": 0.03176403857061826,
+      "grad_norm": 6.46966552734375,
+      "learning_rate": 1.714641594781347e-05,
+      "loss": 7.2163,
+      "step": 294
+    },
+    {
+      "epoch": 0.03187207951813737,
+      "grad_norm": 6.053075790405273,
+      "learning_rate": 1.684386708796025e-05,
+      "loss": 6.8105,
+      "step": 295
+    },
+    {
+      "epoch": 0.031980120465656486,
+      "grad_norm": 5.967854976654053,
+      "learning_rate": 1.6543469682057106e-05,
+      "loss": 7.4358,
+      "step": 296
+    },
+    {
+      "epoch": 0.032088161413175595,
+      "grad_norm": 8.59872055053711,
+      "learning_rate": 1.62452432224612e-05,
+      "loss": 7.5239,
+      "step": 297
+    },
+    {
+      "epoch": 0.0321962023606947,
+      "grad_norm": 6.08354377746582,
+      "learning_rate": 1.5949207060660138e-05,
+      "loss": 6.5361,
+      "step": 298
+    },
+    {
+      "epoch": 0.03230424330821381,
+      "grad_norm": 6.237763404846191,
+      "learning_rate": 1.5655380406016235e-05,
+      "loss": 7.4958,
+      "step": 299
+    },
+    {
+      "epoch": 0.03241228425573292,
+      "grad_norm": 6.916268348693848,
+      "learning_rate": 1.536378232452003e-05,
+      "loss": 7.5241,
+      "step": 300
+    },
+    {
+      "epoch": 0.03241228425573292,
+      "eval_loss": 2.053741931915283,
+      "eval_runtime": 115.0523,
+      "eval_samples_per_second": 135.495,
+      "eval_steps_per_second": 33.88,
+      "step": 300
+    },
+    {
+      "epoch": 0.032520325203252036,
+      "grad_norm": 6.254822254180908,
+      "learning_rate": 1.5074431737553157e-05,
+      "loss": 9.3599,
+      "step": 301
+    },
+    {
+      "epoch": 0.032628366150771144,
+      "grad_norm": 6.089899063110352,
+      "learning_rate": 1.4787347420660541e-05,
+      "loss": 8.4131,
+      "step": 302
+    },
+    {
+      "epoch": 0.03273640709829025,
+      "grad_norm": 5.919285297393799,
+      "learning_rate": 1.4502548002332088e-05,
+      "loss": 8.6915,
+      "step": 303
+    },
+    {
+      "epoch": 0.03284444804580936,
+      "grad_norm": 6.067819118499756,
+      "learning_rate": 1.422005196279395e-05,
+      "loss": 9.2732,
+      "step": 304
+    },
+    {
+      "epoch": 0.03295248899332847,
+      "grad_norm": 6.743030548095703,
+      "learning_rate": 1.3939877632809278e-05,
+      "loss": 8.751,
+      "step": 305
+    },
+    {
+      "epoch": 0.03306052994084758,
+      "grad_norm": 6.73060941696167,
+      "learning_rate": 1.3662043192488849e-05,
+      "loss": 9.1521,
+      "step": 306
+    },
+    {
+      "epoch": 0.033168570888366694,
+      "grad_norm": 6.7592692375183105,
+      "learning_rate": 1.338656667011134e-05,
+      "loss": 9.3304,
+      "step": 307
+    },
+    {
+      "epoch": 0.0332766118358858,
+      "grad_norm": 6.547572612762451,
+      "learning_rate": 1.3113465940953495e-05,
+      "loss": 9.2182,
+      "step": 308
+    },
+    {
+      "epoch": 0.03338465278340491,
+      "grad_norm": 6.383881092071533,
+      "learning_rate": 1.2842758726130283e-05,
+      "loss": 9.5055,
+      "step": 309
+    },
+    {
+      "epoch": 0.03349269373092402,
+      "grad_norm": 6.692324638366699,
+      "learning_rate": 1.257446259144494e-05,
+      "loss": 8.9902,
+      "step": 310
+    },
+    {
+      "epoch": 0.03360073467844313,
+      "grad_norm": 7.0665693283081055,
+      "learning_rate": 1.2308594946249163e-05,
+      "loss": 8.8019,
+      "step": 311
+    },
+    {
+      "epoch": 0.03370877562596224,
+      "grad_norm": 7.003227710723877,
+      "learning_rate": 1.204517304231343e-05,
+      "loss": 8.8641,
+      "step": 312
+    },
+    {
+      "epoch": 0.03381681657348135,
+      "grad_norm": 7.349424362182617,
+      "learning_rate": 1.178421397270758e-05,
+      "loss": 8.4573,
+      "step": 313
+    },
+    {
+      "epoch": 0.03392485752100046,
+      "grad_norm": 8.244421005249023,
+      "learning_rate": 1.1525734670691701e-05,
+      "loss": 10.0232,
+      "step": 314
+    },
+    {
+      "epoch": 0.03403289846851957,
+      "grad_norm": 8.246767044067383,
+      "learning_rate": 1.1269751908617277e-05,
+      "loss": 10.3621,
+      "step": 315
+    },
+    {
+      "epoch": 0.03414093941603868,
+      "grad_norm": 6.383902549743652,
+      "learning_rate": 1.1016282296838887e-05,
+      "loss": 8.0012,
+      "step": 316
+    },
+    {
+      "epoch": 0.034248980363557786,
+      "grad_norm": 5.878174781799316,
+      "learning_rate": 1.0765342282636416e-05,
+      "loss": 7.9243,
+      "step": 317
+    },
+    {
+      "epoch": 0.034357021311076895,
+      "grad_norm": 6.218156337738037,
+      "learning_rate": 1.0516948149147754e-05,
+      "loss": 8.6523,
+      "step": 318
+    },
+    {
+      "epoch": 0.03446506225859601,
+      "grad_norm": 6.337827682495117,
+      "learning_rate": 1.0271116014312293e-05,
+      "loss": 8.2696,
+      "step": 319
+    },
+    {
+      "epoch": 0.03457310320611512,
+      "grad_norm": 6.41033411026001,
+      "learning_rate": 1.0027861829824952e-05,
+      "loss": 7.7903,
+      "step": 320
+    },
+    {
+      "epoch": 0.03468114415363423,
+      "grad_norm": 6.240384578704834,
+      "learning_rate": 9.787201380101157e-06,
+      "loss": 8.5925,
+      "step": 321
+    },
+    {
+      "epoch": 0.034789185101153336,
+      "grad_norm": 7.1170268058776855,
+      "learning_rate": 9.549150281252633e-06,
+      "loss": 9.1973,
+      "step": 322
+    },
+    {
+      "epoch": 0.034897226048672444,
+      "grad_norm": 5.730892181396484,
+      "learning_rate": 9.313723980074018e-06,
+      "loss": 8.3811,
+      "step": 323
+    },
+    {
+      "epoch": 0.03500526699619156,
+      "grad_norm": 6.933986663818359,
+      "learning_rate": 9.080937753040646e-06,
+      "loss": 8.3901,
+      "step": 324
+    },
+    {
+      "epoch": 0.03511330794371067,
+      "grad_norm": 5.97364616394043,
+      "learning_rate": 8.850806705317183e-06,
+      "loss": 8.2894,
+      "step": 325
+    },
+    {
+      "epoch": 0.03522134889122978,
+      "grad_norm": 5.931866645812988,
+      "learning_rate": 8.623345769777514e-06,
+      "loss": 7.9003,
+      "step": 326
+    },
+    {
+      "epoch": 0.035329389838748886,
+      "grad_norm": 5.838757514953613,
+      "learning_rate": 8.398569706035792e-06,
+      "loss": 8.0021,
+      "step": 327
+    },
+    {
+      "epoch": 0.035437430786267994,
+      "grad_norm": 6.679520130157471,
+      "learning_rate": 8.176493099488663e-06,
+      "loss": 8.1413,
+      "step": 328
+    },
+    {
+      "epoch": 0.0355454717337871,
+      "grad_norm": 5.875443935394287,
+      "learning_rate": 7.957130360368898e-06,
+      "loss": 7.6731,
+      "step": 329
+    },
+    {
+      "epoch": 0.03565351268130622,
+      "grad_norm": 6.016922950744629,
+      "learning_rate": 7.740495722810271e-06,
+      "loss": 8.3983,
+      "step": 330
+    },
+    {
+      "epoch": 0.03576155362882533,
+      "grad_norm": 5.992407321929932,
+      "learning_rate": 7.526603243923957e-06,
+      "loss": 7.9567,
+      "step": 331
+    },
+    {
+      "epoch": 0.035869594576344435,
+      "grad_norm": 5.880970478057861,
+      "learning_rate": 7.315466802886401e-06,
+      "loss": 7.2359,
+      "step": 332
+    },
+    {
+      "epoch": 0.035977635523863544,
+      "grad_norm": 5.79258918762207,
+      "learning_rate": 7.107100100038671e-06,
+      "loss": 7.7145,
+      "step": 333
+    },
+    {
+      "epoch": 0.03608567647138265,
+      "grad_norm": 5.844397068023682,
+      "learning_rate": 6.901516655997536e-06,
+      "loss": 7.9453,
+      "step": 334
+    },
+    {
+      "epoch": 0.03619371741890176,
+      "grad_norm": 5.7364959716796875,
+      "learning_rate": 6.698729810778065e-06,
+      "loss": 8.5544,
+      "step": 335
+    },
+    {
+      "epoch": 0.036301758366420876,
+      "grad_norm": 6.327230453491211,
+      "learning_rate": 6.498752722928042e-06,
+      "loss": 8.5004,
+      "step": 336
+    },
+    {
+      "epoch": 0.036409799313939985,
+      "grad_norm": 5.931788444519043,
+      "learning_rate": 6.301598368674105e-06,
+      "loss": 7.5566,
+      "step": 337
+    },
+    {
+      "epoch": 0.03651784026145909,
+      "grad_norm": 6.943851470947266,
+      "learning_rate": 6.107279541079769e-06,
+      "loss": 8.8721,
+      "step": 338
+    },
+    {
+      "epoch": 0.0366258812089782,
+      "grad_norm": 5.852832317352295,
+      "learning_rate": 5.915808849215304e-06,
+      "loss": 7.4767,
+      "step": 339
+    },
+    {
+      "epoch": 0.03673392215649731,
+      "grad_norm": 6.5082831382751465,
+      "learning_rate": 5.727198717339511e-06,
+      "loss": 7.6312,
+      "step": 340
+    },
+    {
+      "epoch": 0.03684196310401642,
+      "grad_norm": 6.173208236694336,
+      "learning_rate": 5.54146138409355e-06,
+      "loss": 7.533,
+      "step": 341
+    },
+    {
+      "epoch": 0.036950004051535534,
+      "grad_norm": 6.1333770751953125,
+      "learning_rate": 5.358608901706802e-06,
+      "loss": 7.5421,
+      "step": 342
+    },
+    {
+      "epoch": 0.03705804499905464,
+      "grad_norm": 5.9275221824646,
+      "learning_rate": 5.178653135214812e-06,
+      "loss": 7.2025,
+      "step": 343
+    },
+    {
+      "epoch": 0.03716608594657375,
+      "grad_norm": 6.223897933959961,
+      "learning_rate": 5.001605761689398e-06,
+      "loss": 7.6112,
+      "step": 344
+    },
+    {
+      "epoch": 0.03727412689409286,
+      "grad_norm": 6.675084114074707,
+      "learning_rate": 4.827478269480895e-06,
+      "loss": 8.3083,
+      "step": 345
+    },
+    {
+      "epoch": 0.03738216784161197,
+      "grad_norm": 6.425478935241699,
+      "learning_rate": 4.65628195747273e-06,
+      "loss": 7.0453,
+      "step": 346
+    },
+    {
+      "epoch": 0.037490208789131084,
+      "grad_norm": 6.075550556182861,
+      "learning_rate": 4.488027934348271e-06,
+      "loss": 7.618,
+      "step": 347
+    },
+    {
+      "epoch": 0.03759824973665019,
+      "grad_norm": 7.088095188140869,
+      "learning_rate": 4.322727117869951e-06,
+      "loss": 7.2647,
+      "step": 348
+    },
+    {
+      "epoch": 0.0377062906841693,
+      "grad_norm": 6.340676307678223,
+      "learning_rate": 4.16039023417088e-06,
+      "loss": 7.4927,
+      "step": 349
+    },
+    {
+      "epoch": 0.03781433163168841,
+      "grad_norm": 6.892800807952881,
+      "learning_rate": 4.001027817058789e-06,
+      "loss": 8.0514,
+      "step": 350
+    },
+    {
+      "epoch": 0.03781433163168841,
+      "eval_loss": 2.0433599948883057,
+      "eval_runtime": 115.1621,
+      "eval_samples_per_second": 135.366,
+      "eval_steps_per_second": 33.848,
+      "step": 350
+    },
+    {
+      "epoch": 0.03792237257920752,
+      "grad_norm": 5.631309509277344,
+      "learning_rate": 3.844650207332562e-06,
+      "loss": 8.7049,
+      "step": 351
+    },
+    {
+      "epoch": 0.03803041352672663,
+      "grad_norm": 5.660338401794434,
+      "learning_rate": 3.691267552111183e-06,
+      "loss": 9.3552,
+      "step": 352
+    },
+    {
+      "epoch": 0.03813845447424574,
+      "grad_norm": 5.268484592437744,
+      "learning_rate": 3.54088980417534e-06,
+      "loss": 8.9509,
+      "step": 353
+    },
+    {
+      "epoch": 0.03824649542176485,
+      "grad_norm": 5.486105442047119,
+      "learning_rate": 3.393526721321616e-06,
+      "loss": 9.506,
+      "step": 354
+    },
+    {
+      "epoch": 0.03835453636928396,
+      "grad_norm": 5.614705562591553,
+      "learning_rate": 3.249187865729264e-06,
+      "loss": 9.3069,
+      "step": 355
+    },
+    {
+      "epoch": 0.03846257731680307,
+      "grad_norm": 5.934595108032227,
+      "learning_rate": 3.1078826033397843e-06,
+      "loss": 9.853,
+      "step": 356
+    },
+    {
+      "epoch": 0.038570618264322176,
+      "grad_norm": 6.226163864135742,
+      "learning_rate": 2.9696201032491434e-06,
+      "loss": 9.2904,
+      "step": 357
+    },
+    {
+      "epoch": 0.038678659211841285,
+      "grad_norm": 6.73871374130249,
+      "learning_rate": 2.8344093371128424e-06,
+      "loss": 9.5113,
+      "step": 358
+    },
+    {
+      "epoch": 0.0387867001593604,
+      "grad_norm": 6.255570411682129,
+      "learning_rate": 2.70225907856374e-06,
+      "loss": 9.411,
+      "step": 359
+    },
+    {
+      "epoch": 0.03889474110687951,
+      "grad_norm": 6.283417224884033,
+      "learning_rate": 2.573177902642726e-06,
+      "loss": 9.4097,
+      "step": 360
+    },
+    {
+      "epoch": 0.03900278205439862,
+      "grad_norm": 6.409605503082275,
+      "learning_rate": 2.4471741852423237e-06,
+      "loss": 9.2033,
+      "step": 361
+    },
+    {
+      "epoch": 0.039110823001917726,
+      "grad_norm": 7.352389335632324,
+      "learning_rate": 2.324256102563188e-06,
+      "loss": 9.4449,
+      "step": 362
+    },
+    {
+      "epoch": 0.039218863949436834,
+      "grad_norm": 7.120124816894531,
+      "learning_rate": 2.204431630583548e-06,
+      "loss": 9.8474,
+      "step": 363
+    },
+    {
+      "epoch": 0.03932690489695594,
+      "grad_norm": 7.766654014587402,
+      "learning_rate": 2.087708544541689e-06,
+      "loss": 9.8473,
+      "step": 364
+    },
+    {
+      "epoch": 0.03943494584447506,
+      "grad_norm": 8.17552661895752,
+      "learning_rate": 1.974094418431388e-06,
+      "loss": 9.0599,
+      "step": 365
+    },
+    {
+      "epoch": 0.03954298679199417,
+      "grad_norm": 8.220863342285156,
+      "learning_rate": 1.8635966245104664e-06,
+      "loss": 9.3122,
+      "step": 366
+    },
+    {
+      "epoch": 0.039651027739513275,
+      "grad_norm": 7.728892803192139,
+      "learning_rate": 1.7562223328224325e-06,
+      "loss": 9.3417,
+      "step": 367
+    },
+    {
+      "epoch": 0.039759068687032384,
+      "grad_norm": 6.223653793334961,
+      "learning_rate": 1.6519785107311891e-06,
+      "loss": 8.3779,
+      "step": 368
+    },
+    {
+      "epoch": 0.03986710963455149,
+      "grad_norm": 6.235572814941406,
+      "learning_rate": 1.5508719224689717e-06,
+      "loss": 8.5487,
+      "step": 369
+    },
+    {
+      "epoch": 0.03997515058207061,
+      "grad_norm": 6.582572937011719,
+      "learning_rate": 1.4529091286973995e-06,
+      "loss": 8.4263,
+      "step": 370
+    },
+    {
+      "epoch": 0.040083191529589716,
+      "grad_norm": 5.888391971588135,
+      "learning_rate": 1.358096486081778e-06,
+      "loss": 8.8294,
+      "step": 371
+    },
+    {
+      "epoch": 0.040191232477108825,
+      "grad_norm": 6.351579666137695,
+      "learning_rate": 1.2664401468786114e-06,
+      "loss": 8.6448,
+      "step": 372
+    },
+    {
+      "epoch": 0.040299273424627934,
+      "grad_norm": 5.644065856933594,
+      "learning_rate": 1.1779460585363944e-06,
+      "loss": 8.2641,
+      "step": 373
+    },
+    {
+      "epoch": 0.04040731437214704,
+      "grad_norm": 5.741602897644043,
+      "learning_rate": 1.0926199633097157e-06,
+      "loss": 8.0143,
+      "step": 374
+    },
+    {
+      "epoch": 0.04051535531966615,
+      "grad_norm": 5.9284257888793945,
+      "learning_rate": 1.0104673978866164e-06,
+      "loss": 8.1583,
+      "step": 375
+    },
+    {
+      "epoch": 0.040623396267185266,
+      "grad_norm": 5.32279109954834,
+      "learning_rate": 9.314936930293283e-07,
+      "loss": 7.3106,
+      "step": 376
+    },
+    {
+      "epoch": 0.040731437214704375,
+      "grad_norm": 5.783631801605225,
+      "learning_rate": 8.557039732283944e-07,
+      "loss": 8.2874,
+      "step": 377
+    },
+    {
+      "epoch": 0.04083947816222348,
+      "grad_norm": 6.149348735809326,
+      "learning_rate": 7.83103156370113e-07,
+      "loss": 8.4637,
+      "step": 378
+    },
+    {
+      "epoch": 0.04094751910974259,
+      "grad_norm": 5.500944137573242,
+      "learning_rate": 7.136959534174592e-07,
+      "loss": 7.5861,
+      "step": 379
+    },
+    {
+      "epoch": 0.0410555600572617,
+      "grad_norm": 5.841290473937988,
+      "learning_rate": 6.474868681043578e-07,
+      "loss": 8.4116,
+      "step": 380
+    },
+    {
+      "epoch": 0.04116360100478081,
+      "grad_norm": 6.573764324188232,
+      "learning_rate": 5.844801966434832e-07,
+      "loss": 8.0255,
+      "step": 381
+    },
+    {
+      "epoch": 0.041271641952299924,
+      "grad_norm": 5.995673656463623,
+      "learning_rate": 5.246800274474439e-07,
+      "loss": 8.016,
+      "step": 382
+    },
+    {
+      "epoch": 0.04137968289981903,
+      "grad_norm": 5.526469707489014,
+      "learning_rate": 4.680902408635335e-07,
+      "loss": 6.731,
+      "step": 383
+    },
+    {
+      "epoch": 0.04148772384733814,
+      "grad_norm": 5.704468727111816,
+      "learning_rate": 4.1471450892189846e-07,
+      "loss": 8.5023,
+      "step": 384
+    },
+    {
+      "epoch": 0.04159576479485725,
+      "grad_norm": 5.787525653839111,
+      "learning_rate": 3.6455629509730136e-07,
+      "loss": 7.3527,
+      "step": 385
+    },
+    {
+      "epoch": 0.04170380574237636,
+      "grad_norm": 5.8779826164245605,
+      "learning_rate": 3.1761885408435054e-07,
+      "loss": 7.3271,
+      "step": 386
+    },
+    {
+      "epoch": 0.041811846689895474,
+      "grad_norm": 6.692193031311035,
+      "learning_rate": 2.7390523158633554e-07,
+      "loss": 8.3191,
+      "step": 387
+    },
+    {
+      "epoch": 0.04191988763741458,
+      "grad_norm": 5.60263729095459,
+      "learning_rate": 2.334182641175686e-07,
+      "loss": 7.3696,
+      "step": 388
+    },
+    {
+      "epoch": 0.04202792858493369,
+      "grad_norm": 5.651635646820068,
+      "learning_rate": 1.9616057881935436e-07,
+      "loss": 7.5317,
+      "step": 389
+    },
+    {
+      "epoch": 0.0421359695324528,
+      "grad_norm": 5.918252468109131,
+      "learning_rate": 1.6213459328950352e-07,
+      "loss": 7.3184,
+      "step": 390
+    },
+    {
+      "epoch": 0.04224401047997191,
+      "grad_norm": 6.036238193511963,
+      "learning_rate": 1.3134251542544774e-07,
+      "loss": 7.8446,
+      "step": 391
+    },
+    {
+      "epoch": 0.042352051427491016,
+      "grad_norm": 6.3520026206970215,
+      "learning_rate": 1.0378634328099269e-07,
+      "loss": 8.1824,
+      "step": 392
+    },
+    {
+      "epoch": 0.04246009237501013,
+      "grad_norm": 6.46418571472168,
+      "learning_rate": 7.946786493666647e-08,
+      "loss": 7.659,
+      "step": 393
+    },
+    {
+      "epoch": 0.04256813332252924,
+      "grad_norm": 5.847161769866943,
+      "learning_rate": 5.838865838366792e-08,
+      "loss": 8.2132,
+      "step": 394
+    },
+    {
+      "epoch": 0.04267617427004835,
+      "grad_norm": 6.146355152130127,
+      "learning_rate": 4.055009142152067e-08,
+      "loss": 7.246,
+      "step": 395
+    },
+    {
+      "epoch": 0.04278421521756746,
+      "grad_norm": 5.900009632110596,
+      "learning_rate": 2.595332156925534e-08,
+      "loss": 7.0707,
+      "step": 396
+    },
+    {
+      "epoch": 0.042892256165086566,
+      "grad_norm": 6.352588176727295,
+      "learning_rate": 1.4599295990352924e-08,
+      "loss": 7.6673,
+      "step": 397
+    },
+    {
+      "epoch": 0.043000297112605675,
+      "grad_norm": 6.337165832519531,
+      "learning_rate": 6.488751431266149e-09,
+      "loss": 7.1714,
+      "step": 398
+    },
+    {
+      "epoch": 0.04310833806012479,
+      "grad_norm": 6.5900983810424805,
+      "learning_rate": 1.622214173602199e-09,
+      "loss": 7.6794,
+      "step": 399
+    },
+    {
+      "epoch": 0.0432163790076439,
+      "grad_norm": 7.042119979858398,
+      "learning_rate": 0.0,
+      "loss": 7.131,
+      "step": 400
+    },
+    {
+      "epoch": 0.0432163790076439,
+      "eval_loss": 2.0413002967834473,
+      "eval_runtime": 115.0871,
+      "eval_samples_per_second": 135.454,
+      "eval_steps_per_second": 33.87,
+      "step": 400
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 400,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 50,
+  "stateful_callbacks": {
+    "EarlyStoppingCallback": {
+      "args": {
+        "early_stopping_patience": 5,
+        "early_stopping_threshold": 0.0
+      },
+      "attributes": {
+        "early_stopping_patience_counter": 0
+      }
+    },
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.0148018821005312e+16,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

last-checkpoint/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12d32a5451ac3f3c82654c2baaa57dd7c0b734caff475a61d51d4f47c5a3fc68
+size 6840

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,31 @@
+{
+  "add_bos_token": true,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "</s>",
+  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "errors": "replace",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "</s>"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12d32a5451ac3f3c82654c2baaa57dd7c0b734caff475a61d51d4f47c5a3fc68
+size 6840