Model save

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,65 @@
+---
+license: gemma
+base_model: google/gemma-2-9b
+tags:
+- trl
+- sft
+- generated_from_trainer
+model-index:
+- name: zephyr-gemma-2-9b-sft-flash
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# zephyr-gemma-2-9b-sft-flash
+
+This model is a fine-tuned version of [google/gemma-2-9b](https://huggingface.co/google/gemma-2-9b) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.8464
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 3e-06
+- train_batch_size: 4
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 128
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss |
+|:-------------:|:-----:|:----:|:---------------:|
+| 0.8289        | 1.0   | 1624 | 0.8464          |
+
+
+### Framework versions
+
+- Transformers 4.45.0.dev0
+- Pytorch 2.4.0+cu121
+- Datasets 2.19.1
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 1278023857274880.0,
+    "train_loss": 0.8594259261864746,
+    "train_runtime": 26490.5415,
+    "train_samples": 207864,
+    "train_samples_per_second": 7.847,
+    "train_steps_per_second": 0.061
+}

config.json

@@ -0,0 +1,34 @@
+{
+  "_name_or_path": "google/gemma-2-9b",
+  "architectures": [
+    "Gemma2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_logit_softcapping": 50.0,
+  "bos_token_id": 2,
+  "cache_implementation": "hybrid",
+  "eos_token_id": 1,
+  "final_logit_softcapping": 30.0,
+  "head_dim": 256,
+  "hidden_act": "gelu_pytorch_tanh",
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 8192,
+  "model_type": "gemma2",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 42,
+  "num_key_value_heads": 8,
+  "pad_token_id": 0,
+  "query_pre_attn_scalar": 256,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "sliding_window_size": 4096,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.45.0.dev0",
+  "use_cache": false,
+  "vocab_size": 256000
+}

generation_config.json

@@ -0,0 +1,8 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 2,
+  "cache_implementation": "hybrid",
+  "eos_token_id": 1,
+  "pad_token_id": 0,
+  "transformers_version": "4.45.0.dev0"
+}

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56a9b123f10fc7fbd19d12cdcb7aa6bae34d3fda887563409aec6843e632c7ce
+size 4903351912

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f026928a45d723b1f6245de08aa55e09597c3f3d789b9fff4d45ab8c0983d4d5
+size 4947570872

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9c87f9ab08df4034b7e9c8816686c70c334d7c7b4c547fad1bcc5b60820f29d
+size 4962221464

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f211134733c546143dfc416eabff3505e846fc18d659b63e784324d0a314382a
+size 3670322200

model.safetensors.index.json

@@ -0,0 +1,471 @@
+{
+  "metadata": {
+    "total_size": 18483411968
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.40.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

runs/Aug09_20-10-30_action-graph-trainer/events.out.tfevents.1723235770.action-graph-trainer.1838021.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3d0360f446397620da7cab873b415315eabd885fb999fd2113c3cbf19a78f2a
+size 7041

runs/Aug09_20-45-13_action-graph-trainer/events.out.tfevents.1723237780.action-graph-trainer.1849282.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77e7412c2073951fcd08597f2922deda5479dddc5c1c159895f6474ec0aabfec
+size 74896

special_tokens_map.json

@@ -0,0 +1,28 @@
+{
+  "additional_special_tokens": [
+    "<start_of_turn>",
+    "<end_of_turn>"
+  ],
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<pad>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d1edfdcd161153437e7cf3968b12b56b335ef420a8db2a75f8e4facd25fbd2b
+size 17525456

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2
+size 4241003

tokenizer_config.json

@@ -0,0 +1,2013 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<eos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<bos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "5": {
+      "content": "<2mass>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "6": {
+      "content": "[@BOS@]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "7": {
+      "content": "<unused0>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "8": {
+      "content": "<unused1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "9": {
+      "content": "<unused2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "10": {
+      "content": "<unused3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "11": {
+      "content": "<unused4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "12": {
+      "content": "<unused5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "13": {
+      "content": "<unused6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "14": {
+      "content": "<unused7>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "15": {
+      "content": "<unused8>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "16": {
+      "content": "<unused9>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "17": {
+      "content": "<unused10>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "18": {
+      "content": "<unused11>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "19": {
+      "content": "<unused12>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "20": {
+      "content": "<unused13>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "21": {
+      "content": "<unused14>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "22": {
+      "content": "<unused15>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "23": {
+      "content": "<unused16>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "24": {
+      "content": "<unused17>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "25": {
+      "content": "<unused18>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "26": {
+      "content": "<unused19>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "27": {
+      "content": "<unused20>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "28": {
+      "content": "<unused21>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "29": {
+      "content": "<unused22>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "30": {
+      "content": "<unused23>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "31": {
+      "content": "<unused24>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "32": {
+      "content": "<unused25>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "33": {
+      "content": "<unused26>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "34": {
+      "content": "<unused27>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "35": {
+      "content": "<unused28>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "36": {
+      "content": "<unused29>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "37": {
+      "content": "<unused30>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "38": {
+      "content": "<unused31>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "39": {
+      "content": "<unused32>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "40": {
+      "content": "<unused33>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "41": {
+      "content": "<unused34>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "42": {
+      "content": "<unused35>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "43": {
+      "content": "<unused36>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "44": {
+      "content": "<unused37>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "45": {
+      "content": "<unused38>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "46": {
+      "content": "<unused39>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "47": {
+      "content": "<unused40>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "48": {
+      "content": "<unused41>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "49": {
+      "content": "<unused42>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50": {
+      "content": "<unused43>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "51": {
+      "content": "<unused44>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "52": {
+      "content": "<unused45>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "53": {
+      "content": "<unused46>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "54": {
+      "content": "<unused47>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "55": {
+      "content": "<unused48>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "56": {
+      "content": "<unused49>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "57": {
+      "content": "<unused50>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "58": {
+      "content": "<unused51>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "59": {
+      "content": "<unused52>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "60": {
+      "content": "<unused53>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "61": {
+      "content": "<unused54>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "62": {
+      "content": "<unused55>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "63": {
+      "content": "<unused56>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "64": {
+      "content": "<unused57>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "65": {
+      "content": "<unused58>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "66": {
+      "content": "<unused59>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "67": {
+      "content": "<unused60>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "68": {
+      "content": "<unused61>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "69": {
+      "content": "<unused62>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "70": {
+      "content": "<unused63>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "71": {
+      "content": "<unused64>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "72": {
+      "content": "<unused65>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "73": {
+      "content": "<unused66>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "74": {
+      "content": "<unused67>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "75": {
+      "content": "<unused68>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "76": {
+      "content": "<unused69>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "77": {
+      "content": "<unused70>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "78": {
+      "content": "<unused71>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "79": {
+      "content": "<unused72>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "80": {
+      "content": "<unused73>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "81": {
+      "content": "<unused74>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "82": {
+      "content": "<unused75>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "83": {
+      "content": "<unused76>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "84": {
+      "content": "<unused77>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "85": {
+      "content": "<unused78>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "86": {
+      "content": "<unused79>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "87": {
+      "content": "<unused80>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "88": {
+      "content": "<unused81>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "89": {
+      "content": "<unused82>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "90": {
+      "content": "<unused83>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "91": {
+      "content": "<unused84>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "92": {
+      "content": "<unused85>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "93": {
+      "content": "<unused86>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "94": {
+      "content": "<unused87>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "95": {
+      "content": "<unused88>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "96": {
+      "content": "<unused89>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "97": {
+      "content": "<unused90>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "98": {
+      "content": "<unused91>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "99": {
+      "content": "<unused92>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100": {
+      "content": "<unused93>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "101": {
+      "content": "<unused94>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "102": {
+      "content": "<unused95>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "103": {
+      "content": "<unused96>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "104": {
+      "content": "<unused97>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "105": {
+      "content": "<unused98>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "106": {
+      "content": "<start_of_turn>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "107": {
+      "content": "<end_of_turn>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "108": {
+      "content": "\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "109": {
+      "content": "\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "110": {
+      "content": "\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "111": {
+      "content": "\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "112": {
+      "content": "\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "113": {
+      "content": "\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "114": {
+      "content": "\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "115": {
+      "content": "\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "116": {
+      "content": "\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "117": {
+      "content": "\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "118": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "119": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "120": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "121": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "122": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "123": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "124": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "125": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "126": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "127": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "128": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "129": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "130": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "131": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "132": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "133": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "134": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "135": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "136": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "137": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "138": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "139": {
+      "content": "▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "140": {
+      "content": "▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "141": {
+      "content": "▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "142": {
+      "content": "▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "143": {
+      "content": "▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "144": {
+      "content": "▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "145": {
+      "content": "▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "146": {
+      "content": "▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "147": {
+      "content": "▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "148": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "149": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "150": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "152": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "153": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "154": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "155": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "156": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "157": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "158": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "159": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "160": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "161": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "162": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "163": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "164": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "165": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "166": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "167": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "168": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "169": {
+      "content": "<table>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "170": {
+      "content": "<caption>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "171": {
+      "content": "<thead>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "172": {
+      "content": "<tbody>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "173": {
+      "content": "<tfoot>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "174": {
+      "content": "<tr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "175": {
+      "content": "<th>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "176": {
+      "content": "<td>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "177": {
+      "content": "</table>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "178": {
+      "content": "</caption>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "179": {
+      "content": "</thead>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "180": {
+      "content": "</tbody>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "181": {
+      "content": "</tfoot>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "182": {
+      "content": "</tr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "183": {
+      "content": "</th>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "184": {
+      "content": "</td>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "185": {
+      "content": "<h1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "186": {
+      "content": "<h2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "187": {
+      "content": "<h3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "188": {
+      "content": "<h4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "189": {
+      "content": "<h5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "190": {
+      "content": "<h6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "191": {
+      "content": "<blockquote>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "192": {
+      "content": "</h1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "193": {
+      "content": "</h2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "194": {
+      "content": "</h3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "195": {
+      "content": "</h4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "196": {
+      "content": "</h5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "197": {
+      "content": "</h6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "198": {
+      "content": "</blockquote>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "199": {
+      "content": "<strong>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "200": {
+      "content": "<em>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "201": {
+      "content": "<b>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "202": {
+      "content": "<i>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "203": {
+      "content": "<u>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "204": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "205": {
+      "content": "<sub>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "206": {
+      "content": "<sup>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "207": {
+      "content": "<code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "208": {
+      "content": "</strong>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "209": {
+      "content": "</em>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "210": {
+      "content": "</b>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "211": {
+      "content": "</i>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "212": {
+      "content": "</u>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "213": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "214": {
+      "content": "</sub>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "215": {
+      "content": "</sup>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "216": {
+      "content": "</code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255968": {
+      "content": "[toxicity=0]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255969": {
+      "content": "\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255970": {
+      "content": "\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255971": {
+      "content": "\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255972": {
+      "content": "\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255973": {
+      "content": "\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255974": {
+      "content": "\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255975": {
+      "content": "\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255976": {
+      "content": "\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255977": {
+      "content": "\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255978": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255979": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255980": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255981": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255982": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255983": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255984": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255985": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255986": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255987": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255988": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255989": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255990": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255991": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255992": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255993": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255994": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255995": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255996": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255997": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255998": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255999": {
+      "content": "<unused99>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<start_of_turn>",
+    "<end_of_turn>"
+  ],
+  "bos_token": "<bos>",
+  "chat_template": "{{ bos_token }}{% for message in messages %}{{ '<start_of_turn>' + message['role'] + '\n' + message['content'] | trim + '<end_of_turn>\n' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model\n'}}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<eos>",
+  "model_max_length": 2048,
+  "pad_token": "<pad>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 1278023857274880.0,
+    "train_loss": 0.8594259261864746,
+    "train_runtime": 26490.5415,
+    "train_samples": 207864,
+    "train_samples_per_second": 7.847,
+    "train_steps_per_second": 0.061
+}

trainer_state.json

@@ -0,0 +1,2325 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 1624,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0006157635467980296,
+      "grad_norm": 6.9261448806343555,
+      "learning_rate": 1.8404907975460124e-08,
+      "loss": 0.9783,
+      "step": 1
+    },
+    {
+      "epoch": 0.003078817733990148,
+      "grad_norm": 7.8540198935077825,
+      "learning_rate": 9.202453987730061e-08,
+      "loss": 1.0023,
+      "step": 5
+    },
+    {
+      "epoch": 0.006157635467980296,
+      "grad_norm": 7.172416861088022,
+      "learning_rate": 1.8404907975460122e-07,
+      "loss": 1.0142,
+      "step": 10
+    },
+    {
+      "epoch": 0.009236453201970444,
+      "grad_norm": 5.363058809753564,
+      "learning_rate": 2.7607361963190183e-07,
+      "loss": 0.9949,
+      "step": 15
+    },
+    {
+      "epoch": 0.012315270935960592,
+      "grad_norm": 2.7182859291547286,
+      "learning_rate": 3.6809815950920245e-07,
+      "loss": 0.9757,
+      "step": 20
+    },
+    {
+      "epoch": 0.01539408866995074,
+      "grad_norm": 4.785438648770351,
+      "learning_rate": 4.601226993865031e-07,
+      "loss": 0.9584,
+      "step": 25
+    },
+    {
+      "epoch": 0.01847290640394089,
+      "grad_norm": 1.9029440372414244,
+      "learning_rate": 5.521472392638037e-07,
+      "loss": 0.934,
+      "step": 30
+    },
+    {
+      "epoch": 0.021551724137931036,
+      "grad_norm": 2.7474733395668514,
+      "learning_rate": 6.441717791411044e-07,
+      "loss": 0.9267,
+      "step": 35
+    },
+    {
+      "epoch": 0.024630541871921183,
+      "grad_norm": 2.2992273802081398,
+      "learning_rate": 7.361963190184049e-07,
+      "loss": 0.9224,
+      "step": 40
+    },
+    {
+      "epoch": 0.02770935960591133,
+      "grad_norm": 1.7323089020577875,
+      "learning_rate": 8.282208588957056e-07,
+      "loss": 0.9006,
+      "step": 45
+    },
+    {
+      "epoch": 0.03078817733990148,
+      "grad_norm": 1.4241841976317904,
+      "learning_rate": 9.202453987730062e-07,
+      "loss": 0.9175,
+      "step": 50
+    },
+    {
+      "epoch": 0.033866995073891626,
+      "grad_norm": 1.2748898710535674,
+      "learning_rate": 1.0122699386503068e-06,
+      "loss": 0.8723,
+      "step": 55
+    },
+    {
+      "epoch": 0.03694581280788178,
+      "grad_norm": 1.3686194000607677,
+      "learning_rate": 1.1042944785276073e-06,
+      "loss": 0.8642,
+      "step": 60
+    },
+    {
+      "epoch": 0.04002463054187192,
+      "grad_norm": 1.3749983673469437,
+      "learning_rate": 1.1963190184049078e-06,
+      "loss": 0.8974,
+      "step": 65
+    },
+    {
+      "epoch": 0.04310344827586207,
+      "grad_norm": 1.359236324741387,
+      "learning_rate": 1.2883435582822088e-06,
+      "loss": 0.882,
+      "step": 70
+    },
+    {
+      "epoch": 0.046182266009852216,
+      "grad_norm": 1.341396608260456,
+      "learning_rate": 1.3803680981595093e-06,
+      "loss": 0.8941,
+      "step": 75
+    },
+    {
+      "epoch": 0.04926108374384237,
+      "grad_norm": 1.3131184545199686,
+      "learning_rate": 1.4723926380368098e-06,
+      "loss": 0.8722,
+      "step": 80
+    },
+    {
+      "epoch": 0.05233990147783251,
+      "grad_norm": 1.4326345843541124,
+      "learning_rate": 1.5644171779141105e-06,
+      "loss": 0.8805,
+      "step": 85
+    },
+    {
+      "epoch": 0.05541871921182266,
+      "grad_norm": 1.3482811175336387,
+      "learning_rate": 1.6564417177914112e-06,
+      "loss": 0.8645,
+      "step": 90
+    },
+    {
+      "epoch": 0.058497536945812806,
+      "grad_norm": 1.339672580787416,
+      "learning_rate": 1.7484662576687115e-06,
+      "loss": 0.8771,
+      "step": 95
+    },
+    {
+      "epoch": 0.06157635467980296,
+      "grad_norm": 1.281527223746865,
+      "learning_rate": 1.8404907975460124e-06,
+      "loss": 0.8878,
+      "step": 100
+    },
+    {
+      "epoch": 0.06465517241379311,
+      "grad_norm": 1.3869101134910244,
+      "learning_rate": 1.932515337423313e-06,
+      "loss": 0.8777,
+      "step": 105
+    },
+    {
+      "epoch": 0.06773399014778325,
+      "grad_norm": 1.2933273902356548,
+      "learning_rate": 2.0245398773006137e-06,
+      "loss": 0.8947,
+      "step": 110
+    },
+    {
+      "epoch": 0.0708128078817734,
+      "grad_norm": 1.3974353379129372,
+      "learning_rate": 2.1165644171779144e-06,
+      "loss": 0.8757,
+      "step": 115
+    },
+    {
+      "epoch": 0.07389162561576355,
+      "grad_norm": 1.335713459694729,
+      "learning_rate": 2.2085889570552147e-06,
+      "loss": 0.8726,
+      "step": 120
+    },
+    {
+      "epoch": 0.0769704433497537,
+      "grad_norm": 1.4532828937018087,
+      "learning_rate": 2.3006134969325154e-06,
+      "loss": 0.8556,
+      "step": 125
+    },
+    {
+      "epoch": 0.08004926108374384,
+      "grad_norm": 1.5819650177294997,
+      "learning_rate": 2.3926380368098157e-06,
+      "loss": 0.8629,
+      "step": 130
+    },
+    {
+      "epoch": 0.08312807881773399,
+      "grad_norm": 1.455259188630674,
+      "learning_rate": 2.4846625766871164e-06,
+      "loss": 0.8849,
+      "step": 135
+    },
+    {
+      "epoch": 0.08620689655172414,
+      "grad_norm": 1.3279627530830256,
+      "learning_rate": 2.5766871165644175e-06,
+      "loss": 0.8895,
+      "step": 140
+    },
+    {
+      "epoch": 0.08928571428571429,
+      "grad_norm": 1.3480166854917446,
+      "learning_rate": 2.668711656441718e-06,
+      "loss": 0.8677,
+      "step": 145
+    },
+    {
+      "epoch": 0.09236453201970443,
+      "grad_norm": 1.4640744251148072,
+      "learning_rate": 2.7607361963190186e-06,
+      "loss": 0.868,
+      "step": 150
+    },
+    {
+      "epoch": 0.09544334975369458,
+      "grad_norm": 1.388612606930217,
+      "learning_rate": 2.852760736196319e-06,
+      "loss": 0.8914,
+      "step": 155
+    },
+    {
+      "epoch": 0.09852216748768473,
+      "grad_norm": 1.2488483163746429,
+      "learning_rate": 2.9447852760736196e-06,
+      "loss": 0.864,
+      "step": 160
+    },
+    {
+      "epoch": 0.10160098522167488,
+      "grad_norm": 1.3620014808461638,
+      "learning_rate": 2.999986128612664e-06,
+      "loss": 0.8787,
+      "step": 165
+    },
+    {
+      "epoch": 0.10467980295566502,
+      "grad_norm": 1.29528505562039,
+      "learning_rate": 2.9998300784514776e-06,
+      "loss": 0.8673,
+      "step": 170
+    },
+    {
+      "epoch": 0.10775862068965517,
+      "grad_norm": 1.3185563525942252,
+      "learning_rate": 2.99950065699352e-06,
+      "loss": 0.8937,
+      "step": 175
+    },
+    {
+      "epoch": 0.11083743842364532,
+      "grad_norm": 1.2075886931677353,
+      "learning_rate": 2.9989979023179235e-06,
+      "loss": 0.8672,
+      "step": 180
+    },
+    {
+      "epoch": 0.11391625615763547,
+      "grad_norm": 1.2553341804143812,
+      "learning_rate": 2.9983218725400893e-06,
+      "loss": 0.8632,
+      "step": 185
+    },
+    {
+      "epoch": 0.11699507389162561,
+      "grad_norm": 1.284600576885917,
+      "learning_rate": 2.9974726458049776e-06,
+      "loss": 0.8703,
+      "step": 190
+    },
+    {
+      "epoch": 0.12007389162561577,
+      "grad_norm": 1.2325841568601283,
+      "learning_rate": 2.9964503202780676e-06,
+      "loss": 0.8522,
+      "step": 195
+    },
+    {
+      "epoch": 0.12315270935960591,
+      "grad_norm": 1.235596663494664,
+      "learning_rate": 2.9952550141340154e-06,
+      "loss": 0.8839,
+      "step": 200
+    },
+    {
+      "epoch": 0.12623152709359606,
+      "grad_norm": 1.491149850603586,
+      "learning_rate": 2.99388686554299e-06,
+      "loss": 0.8876,
+      "step": 205
+    },
+    {
+      "epoch": 0.12931034482758622,
+      "grad_norm": 1.2996245255725345,
+      "learning_rate": 2.9923460326547038e-06,
+      "loss": 0.854,
+      "step": 210
+    },
+    {
+      "epoch": 0.13238916256157635,
+      "grad_norm": 1.3192025851759592,
+      "learning_rate": 2.990632693580131e-06,
+      "loss": 0.8717,
+      "step": 215
+    },
+    {
+      "epoch": 0.1354679802955665,
+      "grad_norm": 1.4480863585669796,
+      "learning_rate": 2.988747046370918e-06,
+      "loss": 0.8681,
+      "step": 220
+    },
+    {
+      "epoch": 0.13854679802955666,
+      "grad_norm": 1.3890044496189642,
+      "learning_rate": 2.986689308996492e-06,
+      "loss": 0.8851,
+      "step": 225
+    },
+    {
+      "epoch": 0.1416256157635468,
+      "grad_norm": 1.2335347167460833,
+      "learning_rate": 2.984459719318862e-06,
+      "loss": 0.8649,
+      "step": 230
+    },
+    {
+      "epoch": 0.14470443349753695,
+      "grad_norm": 1.2655651585410825,
+      "learning_rate": 2.9820585350651265e-06,
+      "loss": 0.8586,
+      "step": 235
+    },
+    {
+      "epoch": 0.1477832512315271,
+      "grad_norm": 1.2714238692165654,
+      "learning_rate": 2.9794860337976802e-06,
+      "loss": 0.8669,
+      "step": 240
+    },
+    {
+      "epoch": 0.15086206896551724,
+      "grad_norm": 1.3072536638433951,
+      "learning_rate": 2.9767425128821288e-06,
+      "loss": 0.8583,
+      "step": 245
+    },
+    {
+      "epoch": 0.1539408866995074,
+      "grad_norm": 1.2521796639409999,
+      "learning_rate": 2.9738282894529177e-06,
+      "loss": 0.874,
+      "step": 250
+    },
+    {
+      "epoch": 0.15701970443349753,
+      "grad_norm": 1.2649805995701187,
+      "learning_rate": 2.9707437003766704e-06,
+      "loss": 0.8676,
+      "step": 255
+    },
+    {
+      "epoch": 0.16009852216748768,
+      "grad_norm": 1.350900196017283,
+      "learning_rate": 2.96748910221325e-06,
+      "loss": 0.8777,
+      "step": 260
+    },
+    {
+      "epoch": 0.16317733990147784,
+      "grad_norm": 1.2861273133536029,
+      "learning_rate": 2.964064871174544e-06,
+      "loss": 0.8677,
+      "step": 265
+    },
+    {
+      "epoch": 0.16625615763546797,
+      "grad_norm": 1.3488102872399326,
+      "learning_rate": 2.9604714030809755e-06,
+      "loss": 0.864,
+      "step": 270
+    },
+    {
+      "epoch": 0.16933497536945813,
+      "grad_norm": 1.2812871832977346,
+      "learning_rate": 2.9567091133157484e-06,
+      "loss": 0.8724,
+      "step": 275
+    },
+    {
+      "epoch": 0.1724137931034483,
+      "grad_norm": 1.2120061211399147,
+      "learning_rate": 2.9527784367768305e-06,
+      "loss": 0.863,
+      "step": 280
+    },
+    {
+      "epoch": 0.17549261083743842,
+      "grad_norm": 1.2468249300241017,
+      "learning_rate": 2.948679827826687e-06,
+      "loss": 0.8772,
+      "step": 285
+    },
+    {
+      "epoch": 0.17857142857142858,
+      "grad_norm": 1.301898471688621,
+      "learning_rate": 2.9444137602397515e-06,
+      "loss": 0.8874,
+      "step": 290
+    },
+    {
+      "epoch": 0.1816502463054187,
+      "grad_norm": 1.2280120813187823,
+      "learning_rate": 2.9399807271476674e-06,
+      "loss": 0.8654,
+      "step": 295
+    },
+    {
+      "epoch": 0.18472906403940886,
+      "grad_norm": 1.2553897731280832,
+      "learning_rate": 2.935381240982281e-06,
+      "loss": 0.8716,
+      "step": 300
+    },
+    {
+      "epoch": 0.18780788177339902,
+      "grad_norm": 1.2466656185114362,
+      "learning_rate": 2.9306158334164066e-06,
+      "loss": 0.8611,
+      "step": 305
+    },
+    {
+      "epoch": 0.19088669950738915,
+      "grad_norm": 1.2434793534776936,
+      "learning_rate": 2.9256850553023724e-06,
+      "loss": 0.8805,
+      "step": 310
+    },
+    {
+      "epoch": 0.1939655172413793,
+      "grad_norm": 1.149005854671108,
+      "learning_rate": 2.920589476608343e-06,
+      "loss": 0.8385,
+      "step": 315
+    },
+    {
+      "epoch": 0.19704433497536947,
+      "grad_norm": 1.1894381020918448,
+      "learning_rate": 2.9153296863524315e-06,
+      "loss": 0.8697,
+      "step": 320
+    },
+    {
+      "epoch": 0.2001231527093596,
+      "grad_norm": 1.3285903611125425,
+      "learning_rate": 2.90990629253462e-06,
+      "loss": 0.8699,
+      "step": 325
+    },
+    {
+      "epoch": 0.20320197044334976,
+      "grad_norm": 1.2825931633975471,
+      "learning_rate": 2.9043199220664704e-06,
+      "loss": 0.8573,
+      "step": 330
+    },
+    {
+      "epoch": 0.2062807881773399,
+      "grad_norm": 1.1444287429011328,
+      "learning_rate": 2.8985712206986627e-06,
+      "loss": 0.8714,
+      "step": 335
+    },
+    {
+      "epoch": 0.20935960591133004,
+      "grad_norm": 1.208641376560723,
+      "learning_rate": 2.8926608529463473e-06,
+      "loss": 0.8658,
+      "step": 340
+    },
+    {
+      "epoch": 0.2124384236453202,
+      "grad_norm": 1.207055661045214,
+      "learning_rate": 2.8865895020123326e-06,
+      "loss": 0.8682,
+      "step": 345
+    },
+    {
+      "epoch": 0.21551724137931033,
+      "grad_norm": 1.191541037712143,
+      "learning_rate": 2.880357869708111e-06,
+      "loss": 0.8959,
+      "step": 350
+    },
+    {
+      "epoch": 0.2185960591133005,
+      "grad_norm": 1.1578477740051558,
+      "learning_rate": 2.8739666763727315e-06,
+      "loss": 0.8696,
+      "step": 355
+    },
+    {
+      "epoch": 0.22167487684729065,
+      "grad_norm": 1.1793751334330371,
+      "learning_rate": 2.8674166607895357e-06,
+      "loss": 0.8627,
+      "step": 360
+    },
+    {
+      "epoch": 0.22475369458128078,
+      "grad_norm": 1.3622878509978327,
+      "learning_rate": 2.8607085801007565e-06,
+      "loss": 0.8546,
+      "step": 365
+    },
+    {
+      "epoch": 0.22783251231527094,
+      "grad_norm": 1.3302148427613436,
+      "learning_rate": 2.85384320972e-06,
+      "loss": 0.8576,
+      "step": 370
+    },
+    {
+      "epoch": 0.2309113300492611,
+      "grad_norm": 1.340634716796683,
+      "learning_rate": 2.846821343242608e-06,
+      "loss": 0.8516,
+      "step": 375
+    },
+    {
+      "epoch": 0.23399014778325122,
+      "grad_norm": 1.2868027249940943,
+      "learning_rate": 2.839643792353928e-06,
+      "loss": 0.8763,
+      "step": 380
+    },
+    {
+      "epoch": 0.23706896551724138,
+      "grad_norm": 1.318614140487259,
+      "learning_rate": 2.832311386735483e-06,
+      "loss": 0.8982,
+      "step": 385
+    },
+    {
+      "epoch": 0.24014778325123154,
+      "grad_norm": 1.2411796540456352,
+      "learning_rate": 2.824824973969069e-06,
+      "loss": 0.8651,
+      "step": 390
+    },
+    {
+      "epoch": 0.24322660098522167,
+      "grad_norm": 1.2756201901720505,
+      "learning_rate": 2.817185419438777e-06,
+      "loss": 0.8666,
+      "step": 395
+    },
+    {
+      "epoch": 0.24630541871921183,
+      "grad_norm": 1.1733814943224672,
+      "learning_rate": 2.8093936062309614e-06,
+      "loss": 0.85,
+      "step": 400
+    },
+    {
+      "epoch": 0.24938423645320196,
+      "grad_norm": 1.2144769750267645,
+      "learning_rate": 2.8014504350321594e-06,
+      "loss": 0.8817,
+      "step": 405
+    },
+    {
+      "epoch": 0.2524630541871921,
+      "grad_norm": 1.2933818324714894,
+      "learning_rate": 2.7933568240249776e-06,
+      "loss": 0.8567,
+      "step": 410
+    },
+    {
+      "epoch": 0.2555418719211823,
+      "grad_norm": 1.2992512099267928,
+      "learning_rate": 2.7851137087819562e-06,
+      "loss": 0.8729,
+      "step": 415
+    },
+    {
+      "epoch": 0.25862068965517243,
+      "grad_norm": 1.2694788614197443,
+      "learning_rate": 2.776722042157421e-06,
+      "loss": 0.8711,
+      "step": 420
+    },
+    {
+      "epoch": 0.2616995073891626,
+      "grad_norm": 1.238206765155676,
+      "learning_rate": 2.768182794177341e-06,
+      "loss": 0.8631,
+      "step": 425
+    },
+    {
+      "epoch": 0.2647783251231527,
+      "grad_norm": 1.2220514457249914,
+      "learning_rate": 2.7594969519271988e-06,
+      "loss": 0.8544,
+      "step": 430
+    },
+    {
+      "epoch": 0.26785714285714285,
+      "grad_norm": 1.287620720175599,
+      "learning_rate": 2.7506655194378874e-06,
+      "loss": 0.8698,
+      "step": 435
+    },
+    {
+      "epoch": 0.270935960591133,
+      "grad_norm": 1.2117242674891486,
+      "learning_rate": 2.7416895175696533e-06,
+      "loss": 0.8787,
+      "step": 440
+    },
+    {
+      "epoch": 0.27401477832512317,
+      "grad_norm": 1.1883277757742343,
+      "learning_rate": 2.7325699838940894e-06,
+      "loss": 0.8807,
+      "step": 445
+    },
+    {
+      "epoch": 0.2770935960591133,
+      "grad_norm": 1.1745648433904117,
+      "learning_rate": 2.723307972574199e-06,
+      "loss": 0.8678,
+      "step": 450
+    },
+    {
+      "epoch": 0.2801724137931034,
+      "grad_norm": 1.21375808638081,
+      "learning_rate": 2.713904554242539e-06,
+      "loss": 0.8659,
+      "step": 455
+    },
+    {
+      "epoch": 0.2832512315270936,
+      "grad_norm": 1.1948010585544386,
+      "learning_rate": 2.7043608158774645e-06,
+      "loss": 0.8598,
+      "step": 460
+    },
+    {
+      "epoch": 0.28633004926108374,
+      "grad_norm": 1.257325488801593,
+      "learning_rate": 2.6946778606774777e-06,
+      "loss": 0.8589,
+      "step": 465
+    },
+    {
+      "epoch": 0.2894088669950739,
+      "grad_norm": 1.2068455301550687,
+      "learning_rate": 2.684856807933706e-06,
+      "loss": 0.8674,
+      "step": 470
+    },
+    {
+      "epoch": 0.29248768472906406,
+      "grad_norm": 1.21613328192186,
+      "learning_rate": 2.6748987929005187e-06,
+      "loss": 0.855,
+      "step": 475
+    },
+    {
+      "epoch": 0.2955665024630542,
+      "grad_norm": 1.2082200902221232,
+      "learning_rate": 2.664804966664298e-06,
+      "loss": 0.887,
+      "step": 480
+    },
+    {
+      "epoch": 0.2986453201970443,
+      "grad_norm": 1.4106856228240403,
+      "learning_rate": 2.6545764960103823e-06,
+      "loss": 0.8699,
+      "step": 485
+    },
+    {
+      "epoch": 0.3017241379310345,
+      "grad_norm": 1.1746699390406088,
+      "learning_rate": 2.6442145632881894e-06,
+      "loss": 0.8688,
+      "step": 490
+    },
+    {
+      "epoch": 0.30480295566502463,
+      "grad_norm": 1.1803466358454566,
+      "learning_rate": 2.6337203662745465e-06,
+      "loss": 0.878,
+      "step": 495
+    },
+    {
+      "epoch": 0.3078817733990148,
+      "grad_norm": 1.2192463459247038,
+      "learning_rate": 2.623095118035235e-06,
+      "loss": 0.8742,
+      "step": 500
+    },
+    {
+      "epoch": 0.31096059113300495,
+      "grad_norm": 1.3450461605379114,
+      "learning_rate": 2.612340046784765e-06,
+      "loss": 0.8785,
+      "step": 505
+    },
+    {
+      "epoch": 0.31403940886699505,
+      "grad_norm": 1.2220476176834498,
+      "learning_rate": 2.601456395744403e-06,
+      "loss": 0.8695,
+      "step": 510
+    },
+    {
+      "epoch": 0.3171182266009852,
+      "grad_norm": 1.2433981390813669,
+      "learning_rate": 2.590445422998462e-06,
+      "loss": 0.8888,
+      "step": 515
+    },
+    {
+      "epoch": 0.32019704433497537,
+      "grad_norm": 1.2086866109912424,
+      "learning_rate": 2.579308401348876e-06,
+      "loss": 0.8667,
+      "step": 520
+    },
+    {
+      "epoch": 0.3232758620689655,
+      "grad_norm": 1.2319324913649576,
+      "learning_rate": 2.56804661816807e-06,
+      "loss": 0.8569,
+      "step": 525
+    },
+    {
+      "epoch": 0.3263546798029557,
+      "grad_norm": 1.1239226127520077,
+      "learning_rate": 2.556661375250149e-06,
+      "loss": 0.8695,
+      "step": 530
+    },
+    {
+      "epoch": 0.3294334975369458,
+      "grad_norm": 1.1017342439849085,
+      "learning_rate": 2.5451539886604165e-06,
+      "loss": 0.8544,
+      "step": 535
+    },
+    {
+      "epoch": 0.33251231527093594,
+      "grad_norm": 1.2174470231839893,
+      "learning_rate": 2.533525788583248e-06,
+      "loss": 0.8681,
+      "step": 540
+    },
+    {
+      "epoch": 0.3355911330049261,
+      "grad_norm": 1.1767214568584958,
+      "learning_rate": 2.5217781191683284e-06,
+      "loss": 0.856,
+      "step": 545
+    },
+    {
+      "epoch": 0.33866995073891626,
+      "grad_norm": 1.1920338504456431,
+      "learning_rate": 2.509912338375275e-06,
+      "loss": 0.849,
+      "step": 550
+    },
+    {
+      "epoch": 0.3417487684729064,
+      "grad_norm": 1.1785002479242088,
+      "learning_rate": 2.497929817816667e-06,
+      "loss": 0.8725,
+      "step": 555
+    },
+    {
+      "epoch": 0.3448275862068966,
+      "grad_norm": 1.1673832474868073,
+      "learning_rate": 2.4858319425994978e-06,
+      "loss": 0.8729,
+      "step": 560
+    },
+    {
+      "epoch": 0.3479064039408867,
+      "grad_norm": 1.2087668323854748,
+      "learning_rate": 2.4736201111650593e-06,
+      "loss": 0.8748,
+      "step": 565
+    },
+    {
+      "epoch": 0.35098522167487683,
+      "grad_norm": 1.2286133334284264,
+      "learning_rate": 2.4612957351272963e-06,
+      "loss": 0.8643,
+      "step": 570
+    },
+    {
+      "epoch": 0.354064039408867,
+      "grad_norm": 1.197334576761319,
+      "learning_rate": 2.448860239109627e-06,
+      "loss": 0.8602,
+      "step": 575
+    },
+    {
+      "epoch": 0.35714285714285715,
+      "grad_norm": 1.2344804890896448,
+      "learning_rate": 2.4363150605802704e-06,
+      "loss": 0.83,
+      "step": 580
+    },
+    {
+      "epoch": 0.3602216748768473,
+      "grad_norm": 1.2074041727344826,
+      "learning_rate": 2.423661649686081e-06,
+      "loss": 0.8735,
+      "step": 585
+    },
+    {
+      "epoch": 0.3633004926108374,
+      "grad_norm": 1.1338699491602573,
+      "learning_rate": 2.41090146908492e-06,
+      "loss": 0.8376,
+      "step": 590
+    },
+    {
+      "epoch": 0.36637931034482757,
+      "grad_norm": 1.1539286741222081,
+      "learning_rate": 2.398035993776582e-06,
+      "loss": 0.8392,
+      "step": 595
+    },
+    {
+      "epoch": 0.3694581280788177,
+      "grad_norm": 1.145963064402236,
+      "learning_rate": 2.385066710932294e-06,
+      "loss": 0.8572,
+      "step": 600
+    },
+    {
+      "epoch": 0.3725369458128079,
+      "grad_norm": 1.2410584143780043,
+      "learning_rate": 2.3719951197228068e-06,
+      "loss": 0.8421,
+      "step": 605
+    },
+    {
+      "epoch": 0.37561576354679804,
+      "grad_norm": 1.169345002551722,
+      "learning_rate": 2.3588227311451007e-06,
+      "loss": 0.8584,
+      "step": 610
+    },
+    {
+      "epoch": 0.3786945812807882,
+      "grad_norm": 1.1663861421904513,
+      "learning_rate": 2.3455510678477217e-06,
+      "loss": 0.8523,
+      "step": 615
+    },
+    {
+      "epoch": 0.3817733990147783,
+      "grad_norm": 1.1951901066607733,
+      "learning_rate": 2.3321816639547747e-06,
+      "loss": 0.8556,
+      "step": 620
+    },
+    {
+      "epoch": 0.38485221674876846,
+      "grad_norm": 1.1634614473774056,
+      "learning_rate": 2.3187160648885864e-06,
+      "loss": 0.8643,
+      "step": 625
+    },
+    {
+      "epoch": 0.3879310344827586,
+      "grad_norm": 1.1754606244319856,
+      "learning_rate": 2.305155827191066e-06,
+      "loss": 0.8591,
+      "step": 630
+    },
+    {
+      "epoch": 0.3910098522167488,
+      "grad_norm": 1.1680093752867005,
+      "learning_rate": 2.291502518343774e-06,
+      "loss": 0.8808,
+      "step": 635
+    },
+    {
+      "epoch": 0.39408866995073893,
+      "grad_norm": 1.200146685343724,
+      "learning_rate": 2.2777577165867354e-06,
+      "loss": 0.8633,
+      "step": 640
+    },
+    {
+      "epoch": 0.39716748768472904,
+      "grad_norm": 1.2302191344741866,
+      "learning_rate": 2.2639230107360033e-06,
+      "loss": 0.8546,
+      "step": 645
+    },
+    {
+      "epoch": 0.4002463054187192,
+      "grad_norm": 1.1972462504203325,
+      "learning_rate": 2.25e-06,
+      "loss": 0.8468,
+      "step": 650
+    },
+    {
+      "epoch": 0.40332512315270935,
+      "grad_norm": 1.2575048017562231,
+      "learning_rate": 2.235990293794659e-06,
+      "loss": 0.8789,
+      "step": 655
+    },
+    {
+      "epoch": 0.4064039408866995,
+      "grad_norm": 1.2254392018354676,
+      "learning_rate": 2.2218955115573864e-06,
+      "loss": 0.8675,
+      "step": 660
+    },
+    {
+      "epoch": 0.40948275862068967,
+      "grad_norm": 1.289673826138273,
+      "learning_rate": 2.2077172825598645e-06,
+      "loss": 0.8542,
+      "step": 665
+    },
+    {
+      "epoch": 0.4125615763546798,
+      "grad_norm": 1.155592523195983,
+      "learning_rate": 2.1934572457197163e-06,
+      "loss": 0.8674,
+      "step": 670
+    },
+    {
+      "epoch": 0.41564039408866993,
+      "grad_norm": 1.118480694129497,
+      "learning_rate": 2.179117049411057e-06,
+      "loss": 0.863,
+      "step": 675
+    },
+    {
+      "epoch": 0.4187192118226601,
+      "grad_norm": 1.196598731794123,
+      "learning_rate": 2.164698351273952e-06,
+      "loss": 0.8607,
+      "step": 680
+    },
+    {
+      "epoch": 0.42179802955665024,
+      "grad_norm": 1.1462380782787314,
+      "learning_rate": 2.150202818022805e-06,
+      "loss": 0.862,
+      "step": 685
+    },
+    {
+      "epoch": 0.4248768472906404,
+      "grad_norm": 1.1934975175320721,
+      "learning_rate": 2.1356321252536947e-06,
+      "loss": 0.849,
+      "step": 690
+    },
+    {
+      "epoch": 0.42795566502463056,
+      "grad_norm": 1.1882656818519586,
+      "learning_rate": 2.1209879572506836e-06,
+      "loss": 0.8612,
+      "step": 695
+    },
+    {
+      "epoch": 0.43103448275862066,
+      "grad_norm": 1.2411770323256186,
+      "learning_rate": 2.10627200679113e-06,
+      "loss": 0.8595,
+      "step": 700
+    },
+    {
+      "epoch": 0.4341133004926108,
+      "grad_norm": 1.2705680290759138,
+      "learning_rate": 2.09148597495001e-06,
+      "loss": 0.8756,
+      "step": 705
+    },
+    {
+      "epoch": 0.437192118226601,
+      "grad_norm": 1.229328701037844,
+      "learning_rate": 2.0766315709032837e-06,
+      "loss": 0.8402,
+      "step": 710
+    },
+    {
+      "epoch": 0.44027093596059114,
+      "grad_norm": 1.19622724937497,
+      "learning_rate": 2.0617105117303247e-06,
+      "loss": 0.8566,
+      "step": 715
+    },
+    {
+      "epoch": 0.4433497536945813,
+      "grad_norm": 1.1456453404254636,
+      "learning_rate": 2.046724522215437e-06,
+      "loss": 0.8436,
+      "step": 720
+    },
+    {
+      "epoch": 0.44642857142857145,
+      "grad_norm": 1.2156871994700378,
+      "learning_rate": 2.03167533464848e-06,
+      "loss": 0.8636,
+      "step": 725
+    },
+    {
+      "epoch": 0.44950738916256155,
+      "grad_norm": 1.1050184006190784,
+      "learning_rate": 2.016564688624627e-06,
+      "loss": 0.8676,
+      "step": 730
+    },
+    {
+      "epoch": 0.4525862068965517,
+      "grad_norm": 1.2054799838127823,
+      "learning_rate": 2.001394330843276e-06,
+      "loss": 0.8371,
+      "step": 735
+    },
+    {
+      "epoch": 0.45566502463054187,
+      "grad_norm": 1.1765001008173661,
+      "learning_rate": 1.9861660149061435e-06,
+      "loss": 0.8698,
+      "step": 740
+    },
+    {
+      "epoch": 0.45874384236453203,
+      "grad_norm": 1.1626940687518046,
+      "learning_rate": 1.9708815011145605e-06,
+      "loss": 0.8607,
+      "step": 745
+    },
+    {
+      "epoch": 0.4618226600985222,
+      "grad_norm": 1.2130798558410487,
+      "learning_rate": 1.9555425562659878e-06,
+      "loss": 0.8633,
+      "step": 750
+    },
+    {
+      "epoch": 0.4649014778325123,
+      "grad_norm": 1.1787298608214307,
+      "learning_rate": 1.940150953449791e-06,
+      "loss": 0.8272,
+      "step": 755
+    },
+    {
+      "epoch": 0.46798029556650245,
+      "grad_norm": 1.1385414505234648,
+      "learning_rate": 1.924708471842276e-06,
+      "loss": 0.8568,
+      "step": 760
+    },
+    {
+      "epoch": 0.4710591133004926,
+      "grad_norm": 1.1442893302994293,
+      "learning_rate": 1.90921689650103e-06,
+      "loss": 0.8586,
+      "step": 765
+    },
+    {
+      "epoch": 0.47413793103448276,
+      "grad_norm": 1.1775293738465633,
+      "learning_rate": 1.8936780181585799e-06,
+      "loss": 0.8479,
+      "step": 770
+    },
+    {
+      "epoch": 0.4772167487684729,
+      "grad_norm": 1.1600712023507014,
+      "learning_rate": 1.8780936330153935e-06,
+      "loss": 0.8683,
+      "step": 775
+    },
+    {
+      "epoch": 0.4802955665024631,
+      "grad_norm": 1.1087281329789505,
+      "learning_rate": 1.86246554253225e-06,
+      "loss": 0.8606,
+      "step": 780
+    },
+    {
+      "epoch": 0.4833743842364532,
+      "grad_norm": 1.197533696719566,
+      "learning_rate": 1.8467955532220017e-06,
+      "loss": 0.8501,
+      "step": 785
+    },
+    {
+      "epoch": 0.48645320197044334,
+      "grad_norm": 1.3268508195213191,
+      "learning_rate": 1.831085476440753e-06,
+      "loss": 0.8528,
+      "step": 790
+    },
+    {
+      "epoch": 0.4895320197044335,
+      "grad_norm": 1.1609022696881155,
+      "learning_rate": 1.8153371281784756e-06,
+      "loss": 0.8563,
+      "step": 795
+    },
+    {
+      "epoch": 0.49261083743842365,
+      "grad_norm": 1.2734002436608558,
+      "learning_rate": 1.7995523288490959e-06,
+      "loss": 0.871,
+      "step": 800
+    },
+    {
+      "epoch": 0.4956896551724138,
+      "grad_norm": 1.2429196155495275,
+      "learning_rate": 1.783732903080062e-06,
+      "loss": 0.8636,
+      "step": 805
+    },
+    {
+      "epoch": 0.4987684729064039,
+      "grad_norm": 1.2216553602881683,
+      "learning_rate": 1.7678806795014293e-06,
+      "loss": 0.8591,
+      "step": 810
+    },
+    {
+      "epoch": 0.5018472906403941,
+      "grad_norm": 1.2317941320359274,
+      "learning_rate": 1.751997490534482e-06,
+      "loss": 0.8561,
+      "step": 815
+    },
+    {
+      "epoch": 0.5049261083743842,
+      "grad_norm": 1.240928548512091,
+      "learning_rate": 1.7360851721799163e-06,
+      "loss": 0.819,
+      "step": 820
+    },
+    {
+      "epoch": 0.5080049261083743,
+      "grad_norm": 1.137834288895378,
+      "learning_rate": 1.7201455638056097e-06,
+      "loss": 0.8471,
+      "step": 825
+    },
+    {
+      "epoch": 0.5110837438423645,
+      "grad_norm": 1.2218216629048144,
+      "learning_rate": 1.7041805079340006e-06,
+      "loss": 0.8517,
+      "step": 830
+    },
+    {
+      "epoch": 0.5141625615763546,
+      "grad_norm": 1.1287151216098226,
+      "learning_rate": 1.6881918500291052e-06,
+      "loss": 0.8669,
+      "step": 835
+    },
+    {
+      "epoch": 0.5172413793103449,
+      "grad_norm": 1.1750153544752535,
+      "learning_rate": 1.6721814382831911e-06,
+      "loss": 0.8333,
+      "step": 840
+    },
+    {
+      "epoch": 0.520320197044335,
+      "grad_norm": 1.3517971320793265,
+      "learning_rate": 1.6561511234031394e-06,
+      "loss": 0.8509,
+      "step": 845
+    },
+    {
+      "epoch": 0.5233990147783252,
+      "grad_norm": 1.1510765432393526,
+      "learning_rate": 1.6401027583965135e-06,
+      "loss": 0.8546,
+      "step": 850
+    },
+    {
+      "epoch": 0.5264778325123153,
+      "grad_norm": 1.145534568185499,
+      "learning_rate": 1.624038198357361e-06,
+      "loss": 0.8602,
+      "step": 855
+    },
+    {
+      "epoch": 0.5295566502463054,
+      "grad_norm": 1.0757414794782645,
+      "learning_rate": 1.6079593002517785e-06,
+      "loss": 0.8873,
+      "step": 860
+    },
+    {
+      "epoch": 0.5326354679802956,
+      "grad_norm": 1.2502085028429488,
+      "learning_rate": 1.5918679227032564e-06,
+      "loss": 0.8464,
+      "step": 865
+    },
+    {
+      "epoch": 0.5357142857142857,
+      "grad_norm": 1.1222279945478513,
+      "learning_rate": 1.575765925777834e-06,
+      "loss": 0.8511,
+      "step": 870
+    },
+    {
+      "epoch": 0.5387931034482759,
+      "grad_norm": 1.186279741537009,
+      "learning_rate": 1.559655170769084e-06,
+      "loss": 0.8505,
+      "step": 875
+    },
+    {
+      "epoch": 0.541871921182266,
+      "grad_norm": 1.1629287769906702,
+      "learning_rate": 1.543537519982963e-06,
+      "loss": 0.8543,
+      "step": 880
+    },
+    {
+      "epoch": 0.5449507389162561,
+      "grad_norm": 1.1376251699369824,
+      "learning_rate": 1.5274148365225372e-06,
+      "loss": 0.8514,
+      "step": 885
+    },
+    {
+      "epoch": 0.5480295566502463,
+      "grad_norm": 1.1193029800588687,
+      "learning_rate": 1.5112889840726194e-06,
+      "loss": 0.8714,
+      "step": 890
+    },
+    {
+      "epoch": 0.5511083743842364,
+      "grad_norm": 1.0765199225592497,
+      "learning_rate": 1.4951618266843384e-06,
+      "loss": 0.8354,
+      "step": 895
+    },
+    {
+      "epoch": 0.5541871921182266,
+      "grad_norm": 1.2664400010011465,
+      "learning_rate": 1.4790352285596656e-06,
+      "loss": 0.8546,
+      "step": 900
+    },
+    {
+      "epoch": 0.5572660098522167,
+      "grad_norm": 1.2761535311975785,
+      "learning_rate": 1.4629110538359251e-06,
+      "loss": 0.843,
+      "step": 905
+    },
+    {
+      "epoch": 0.5603448275862069,
+      "grad_norm": 1.1858099028061224,
+      "learning_rate": 1.4467911663703118e-06,
+      "loss": 0.8446,
+      "step": 910
+    },
+    {
+      "epoch": 0.5634236453201971,
+      "grad_norm": 1.1349117902212704,
+      "learning_rate": 1.4306774295244372e-06,
+      "loss": 0.8574,
+      "step": 915
+    },
+    {
+      "epoch": 0.5665024630541872,
+      "grad_norm": 1.1354130154448216,
+      "learning_rate": 1.4145717059489405e-06,
+      "loss": 0.8554,
+      "step": 920
+    },
+    {
+      "epoch": 0.5695812807881774,
+      "grad_norm": 1.1101009842333847,
+      "learning_rate": 1.3984758573681741e-06,
+      "loss": 0.8371,
+      "step": 925
+    },
+    {
+      "epoch": 0.5726600985221675,
+      "grad_norm": 1.2188838518881486,
+      "learning_rate": 1.3823917443649994e-06,
+      "loss": 0.8253,
+      "step": 930
+    },
+    {
+      "epoch": 0.5757389162561576,
+      "grad_norm": 1.1543850666653297,
+      "learning_rate": 1.3663212261657191e-06,
+      "loss": 0.8529,
+      "step": 935
+    },
+    {
+      "epoch": 0.5788177339901478,
+      "grad_norm": 1.1882880368282511,
+      "learning_rate": 1.3502661604251562e-06,
+      "loss": 0.8511,
+      "step": 940
+    },
+    {
+      "epoch": 0.5818965517241379,
+      "grad_norm": 1.102823629661978,
+      "learning_rate": 1.3342284030119233e-06,
+      "loss": 0.8386,
+      "step": 945
+    },
+    {
+      "epoch": 0.5849753694581281,
+      "grad_norm": 1.100022435336952,
+      "learning_rate": 1.3182098077938954e-06,
+      "loss": 0.8396,
+      "step": 950
+    },
+    {
+      "epoch": 0.5880541871921182,
+      "grad_norm": 1.1581491148327594,
+      "learning_rate": 1.3022122264239134e-06,
+      "loss": 0.8683,
+      "step": 955
+    },
+    {
+      "epoch": 0.5911330049261084,
+      "grad_norm": 1.1371498189322844,
+      "learning_rate": 1.286237508125744e-06,
+      "loss": 0.8648,
+      "step": 960
+    },
+    {
+      "epoch": 0.5942118226600985,
+      "grad_norm": 1.1237133598403473,
+      "learning_rate": 1.2702874994803206e-06,
+      "loss": 0.8568,
+      "step": 965
+    },
+    {
+      "epoch": 0.5972906403940886,
+      "grad_norm": 1.2164340592110872,
+      "learning_rate": 1.254364044212291e-06,
+      "loss": 0.8603,
+      "step": 970
+    },
+    {
+      "epoch": 0.6003694581280788,
+      "grad_norm": 1.195502749881507,
+      "learning_rate": 1.2384689829768929e-06,
+      "loss": 0.8332,
+      "step": 975
+    },
+    {
+      "epoch": 0.603448275862069,
+      "grad_norm": 1.1883714528871503,
+      "learning_rate": 1.2226041531471835e-06,
+      "loss": 0.8608,
+      "step": 980
+    },
+    {
+      "epoch": 0.6065270935960592,
+      "grad_norm": 1.221532799323495,
+      "learning_rate": 1.2067713886016547e-06,
+      "loss": 0.8345,
+      "step": 985
+    },
+    {
+      "epoch": 0.6096059113300493,
+      "grad_norm": 1.0665999327801376,
+      "learning_rate": 1.1909725195122443e-06,
+      "loss": 0.854,
+      "step": 990
+    },
+    {
+      "epoch": 0.6126847290640394,
+      "grad_norm": 1.0972038330985938,
+      "learning_rate": 1.1752093721327784e-06,
+      "loss": 0.8384,
+      "step": 995
+    },
+    {
+      "epoch": 0.6157635467980296,
+      "grad_norm": 1.0800229293813715,
+      "learning_rate": 1.1594837685878725e-06,
+      "loss": 0.8225,
+      "step": 1000
+    },
+    {
+      "epoch": 0.6188423645320197,
+      "grad_norm": 1.2108155943977397,
+      "learning_rate": 1.143797526662299e-06,
+      "loss": 0.8706,
+      "step": 1005
+    },
+    {
+      "epoch": 0.6219211822660099,
+      "grad_norm": 1.1329492061465034,
+      "learning_rate": 1.1281524595908653e-06,
+      "loss": 0.8363,
+      "step": 1010
+    },
+    {
+      "epoch": 0.625,
+      "grad_norm": 1.140246295184195,
+      "learning_rate": 1.1125503758488158e-06,
+      "loss": 0.8569,
+      "step": 1015
+    },
+    {
+      "epoch": 0.6280788177339901,
+      "grad_norm": 1.1714239549738437,
+      "learning_rate": 1.0969930789427798e-06,
+      "loss": 0.852,
+      "step": 1020
+    },
+    {
+      "epoch": 0.6311576354679803,
+      "grad_norm": 1.1649482511304887,
+      "learning_rate": 1.0814823672023007e-06,
+      "loss": 0.8441,
+      "step": 1025
+    },
+    {
+      "epoch": 0.6342364532019704,
+      "grad_norm": 1.1653119573155162,
+      "learning_rate": 1.0660200335719569e-06,
+      "loss": 0.8637,
+      "step": 1030
+    },
+    {
+      "epoch": 0.6373152709359606,
+      "grad_norm": 1.1748817907592242,
+      "learning_rate": 1.0506078654041095e-06,
+      "loss": 0.8548,
+      "step": 1035
+    },
+    {
+      "epoch": 0.6403940886699507,
+      "grad_norm": 1.150982101534324,
+      "learning_rate": 1.0352476442522963e-06,
+      "loss": 0.8666,
+      "step": 1040
+    },
+    {
+      "epoch": 0.6434729064039408,
+      "grad_norm": 1.19598068262744,
+      "learning_rate": 1.0199411456652932e-06,
+      "loss": 0.8346,
+      "step": 1045
+    },
+    {
+      "epoch": 0.646551724137931,
+      "grad_norm": 1.1155499164869085,
+      "learning_rate": 1.004690138981871e-06,
+      "loss": 0.8679,
+      "step": 1050
+    },
+    {
+      "epoch": 0.6496305418719212,
+      "grad_norm": 1.1153195892934271,
+      "learning_rate": 9.89496387126274e-07,
+      "loss": 0.8495,
+      "step": 1055
+    },
+    {
+      "epoch": 0.6527093596059114,
+      "grad_norm": 1.1438524231577982,
+      "learning_rate": 9.74361646404432e-07,
+      "loss": 0.8247,
+      "step": 1060
+    },
+    {
+      "epoch": 0.6557881773399015,
+      "grad_norm": 1.2072652809692634,
+      "learning_rate": 9.592876663009462e-07,
+      "loss": 0.8535,
+      "step": 1065
+    },
+    {
+      "epoch": 0.6588669950738916,
+      "grad_norm": 1.1448538596718278,
+      "learning_rate": 9.442761892768561e-07,
+      "loss": 0.8275,
+      "step": 1070
+    },
+    {
+      "epoch": 0.6619458128078818,
+      "grad_norm": 1.1595389631526323,
+      "learning_rate": 9.293289505682223e-07,
+      "loss": 0.8619,
+      "step": 1075
+    },
+    {
+      "epoch": 0.6650246305418719,
+      "grad_norm": 1.168619612056998,
+      "learning_rate": 9.144476779855462e-07,
+      "loss": 0.8721,
+      "step": 1080
+    },
+    {
+      "epoch": 0.6681034482758621,
+      "grad_norm": 1.1419637237675953,
+      "learning_rate": 8.99634091714042e-07,
+      "loss": 0.8571,
+      "step": 1085
+    },
+    {
+      "epoch": 0.6711822660098522,
+      "grad_norm": 1.0997834592491753,
+      "learning_rate": 8.848899041147947e-07,
+      "loss": 0.8452,
+      "step": 1090
+    },
+    {
+      "epoch": 0.6742610837438424,
+      "grad_norm": 1.0451607430776064,
+      "learning_rate": 8.702168195268239e-07,
+      "loss": 0.8386,
+      "step": 1095
+    },
+    {
+      "epoch": 0.6773399014778325,
+      "grad_norm": 1.1125423600709152,
+      "learning_rate": 8.556165340700687e-07,
+      "loss": 0.8407,
+      "step": 1100
+    },
+    {
+      "epoch": 0.6804187192118226,
+      "grad_norm": 1.201714593633653,
+      "learning_rate": 8.410907354493304e-07,
+      "loss": 0.8394,
+      "step": 1105
+    },
+    {
+      "epoch": 0.6834975369458128,
+      "grad_norm": 1.1545196446483614,
+      "learning_rate": 8.266411027591801e-07,
+      "loss": 0.8589,
+      "step": 1110
+    },
+    {
+      "epoch": 0.6865763546798029,
+      "grad_norm": 1.1591807153074163,
+      "learning_rate": 8.12269306289868e-07,
+      "loss": 0.8465,
+      "step": 1115
+    },
+    {
+      "epoch": 0.6896551724137931,
+      "grad_norm": 1.2440832397679722,
+      "learning_rate": 7.979770073342484e-07,
+      "loss": 0.8556,
+      "step": 1120
+    },
+    {
+      "epoch": 0.6927339901477833,
+      "grad_norm": 1.1921056753242245,
+      "learning_rate": 7.837658579957422e-07,
+      "loss": 0.8419,
+      "step": 1125
+    },
+    {
+      "epoch": 0.6958128078817734,
+      "grad_norm": 1.120932283202893,
+      "learning_rate": 7.696375009973643e-07,
+      "loss": 0.8311,
+      "step": 1130
+    },
+    {
+      "epoch": 0.6988916256157636,
+      "grad_norm": 1.1242115774431858,
+      "learning_rate": 7.555935694918371e-07,
+      "loss": 0.8554,
+      "step": 1135
+    },
+    {
+      "epoch": 0.7019704433497537,
+      "grad_norm": 1.1118916447449778,
+      "learning_rate": 7.41635686872804e-07,
+      "loss": 0.8524,
+      "step": 1140
+    },
+    {
+      "epoch": 0.7050492610837439,
+      "grad_norm": 1.0675374925364602,
+      "learning_rate": 7.277654665871785e-07,
+      "loss": 0.8224,
+      "step": 1145
+    },
+    {
+      "epoch": 0.708128078817734,
+      "grad_norm": 1.1754673704008272,
+      "learning_rate": 7.139845119486371e-07,
+      "loss": 0.8249,
+      "step": 1150
+    },
+    {
+      "epoch": 0.7112068965517241,
+      "grad_norm": 1.1549260560209382,
+      "learning_rate": 7.00294415952286e-07,
+      "loss": 0.851,
+      "step": 1155
+    },
+    {
+      "epoch": 0.7142857142857143,
+      "grad_norm": 1.2059181376557389,
+      "learning_rate": 6.866967610905234e-07,
+      "loss": 0.8593,
+      "step": 1160
+    },
+    {
+      "epoch": 0.7173645320197044,
+      "grad_norm": 1.168368680970263,
+      "learning_rate": 6.731931191701088e-07,
+      "loss": 0.8405,
+      "step": 1165
+    },
+    {
+      "epoch": 0.7204433497536946,
+      "grad_norm": 1.1083640031556827,
+      "learning_rate": 6.597850511304739e-07,
+      "loss": 0.8304,
+      "step": 1170
+    },
+    {
+      "epoch": 0.7235221674876847,
+      "grad_norm": 1.1385014460598872,
+      "learning_rate": 6.464741068632883e-07,
+      "loss": 0.8348,
+      "step": 1175
+    },
+    {
+      "epoch": 0.7266009852216748,
+      "grad_norm": 1.1754958972549083,
+      "learning_rate": 6.332618250332988e-07,
+      "loss": 0.8242,
+      "step": 1180
+    },
+    {
+      "epoch": 0.729679802955665,
+      "grad_norm": 1.1537062502426862,
+      "learning_rate": 6.201497329004718e-07,
+      "loss": 0.863,
+      "step": 1185
+    },
+    {
+      "epoch": 0.7327586206896551,
+      "grad_norm": 1.1522616135684283,
+      "learning_rate": 6.071393461434488e-07,
+      "loss": 0.8469,
+      "step": 1190
+    },
+    {
+      "epoch": 0.7358374384236454,
+      "grad_norm": 1.1588974844109057,
+      "learning_rate": 5.942321686843444e-07,
+      "loss": 0.8352,
+      "step": 1195
+    },
+    {
+      "epoch": 0.7389162561576355,
+      "grad_norm": 1.1149479873307793,
+      "learning_rate": 5.814296925149026e-07,
+      "loss": 0.8666,
+      "step": 1200
+    },
+    {
+      "epoch": 0.7419950738916257,
+      "grad_norm": 1.161774539525719,
+      "learning_rate": 5.687333975240304e-07,
+      "loss": 0.8417,
+      "step": 1205
+    },
+    {
+      "epoch": 0.7450738916256158,
+      "grad_norm": 1.1895380482820657,
+      "learning_rate": 5.561447513267311e-07,
+      "loss": 0.8544,
+      "step": 1210
+    },
+    {
+      "epoch": 0.7481527093596059,
+      "grad_norm": 1.0812447752197967,
+      "learning_rate": 5.436652090944596e-07,
+      "loss": 0.8335,
+      "step": 1215
+    },
+    {
+      "epoch": 0.7512315270935961,
+      "grad_norm": 1.0978213936781354,
+      "learning_rate": 5.312962133869093e-07,
+      "loss": 0.8529,
+      "step": 1220
+    },
+    {
+      "epoch": 0.7543103448275862,
+      "grad_norm": 1.1752412136263564,
+      "learning_rate": 5.19039193985266e-07,
+      "loss": 0.8335,
+      "step": 1225
+    },
+    {
+      "epoch": 0.7573891625615764,
+      "grad_norm": 1.1400037969508205,
+      "learning_rate": 5.068955677269281e-07,
+      "loss": 0.8453,
+      "step": 1230
+    },
+    {
+      "epoch": 0.7604679802955665,
+      "grad_norm": 1.0578584004266824,
+      "learning_rate": 4.948667383417332e-07,
+      "loss": 0.8618,
+      "step": 1235
+    },
+    {
+      "epoch": 0.7635467980295566,
+      "grad_norm": 1.1215254584597703,
+      "learning_rate": 4.829540962896927e-07,
+      "loss": 0.8459,
+      "step": 1240
+    },
+    {
+      "epoch": 0.7666256157635468,
+      "grad_norm": 1.1501053088350557,
+      "learning_rate": 4.7115901860026634e-07,
+      "loss": 0.8553,
+      "step": 1245
+    },
+    {
+      "epoch": 0.7697044334975369,
+      "grad_norm": 1.1101973479276739,
+      "learning_rate": 4.594828687131814e-07,
+      "loss": 0.8421,
+      "step": 1250
+    },
+    {
+      "epoch": 0.7727832512315271,
+      "grad_norm": 1.150410053744879,
+      "learning_rate": 4.4792699632083043e-07,
+      "loss": 0.8339,
+      "step": 1255
+    },
+    {
+      "epoch": 0.7758620689655172,
+      "grad_norm": 1.101120589443304,
+      "learning_rate": 4.36492737212255e-07,
+      "loss": 0.8498,
+      "step": 1260
+    },
+    {
+      "epoch": 0.7789408866995073,
+      "grad_norm": 1.106272741222952,
+      "learning_rate": 4.251814131187349e-07,
+      "loss": 0.8332,
+      "step": 1265
+    },
+    {
+      "epoch": 0.7820197044334976,
+      "grad_norm": 1.1744175651801672,
+      "learning_rate": 4.13994331561004e-07,
+      "loss": 0.8292,
+      "step": 1270
+    },
+    {
+      "epoch": 0.7850985221674877,
+      "grad_norm": 1.0751076819848788,
+      "learning_rate": 4.0293278569811197e-07,
+      "loss": 0.8358,
+      "step": 1275
+    },
+    {
+      "epoch": 0.7881773399014779,
+      "grad_norm": 1.139204040020546,
+      "learning_rate": 3.9199805417793833e-07,
+      "loss": 0.8491,
+      "step": 1280
+    },
+    {
+      "epoch": 0.791256157635468,
+      "grad_norm": 1.1657897926157543,
+      "learning_rate": 3.8119140098939074e-07,
+      "loss": 0.8387,
+      "step": 1285
+    },
+    {
+      "epoch": 0.7943349753694581,
+      "grad_norm": 1.1171782960279335,
+      "learning_rate": 3.705140753162973e-07,
+      "loss": 0.8673,
+      "step": 1290
+    },
+    {
+      "epoch": 0.7974137931034483,
+      "grad_norm": 1.1290132452277677,
+      "learning_rate": 3.5996731139300476e-07,
+      "loss": 0.841,
+      "step": 1295
+    },
+    {
+      "epoch": 0.8004926108374384,
+      "grad_norm": 1.1465611245119605,
+      "learning_rate": 3.495523283617106e-07,
+      "loss": 0.851,
+      "step": 1300
+    },
+    {
+      "epoch": 0.8035714285714286,
+      "grad_norm": 1.135249879261848,
+      "learning_rate": 3.39270330131538e-07,
+      "loss": 0.84,
+      "step": 1305
+    },
+    {
+      "epoch": 0.8066502463054187,
+      "grad_norm": 1.277397410730925,
+      "learning_rate": 3.2912250523937e-07,
+      "loss": 0.8553,
+      "step": 1310
+    },
+    {
+      "epoch": 0.8097290640394089,
+      "grad_norm": 1.1709150489167435,
+      "learning_rate": 3.1911002671246164e-07,
+      "loss": 0.8406,
+      "step": 1315
+    },
+    {
+      "epoch": 0.812807881773399,
+      "grad_norm": 1.1195982285901722,
+      "learning_rate": 3.092340519328474e-07,
+      "loss": 0.8845,
+      "step": 1320
+    },
+    {
+      "epoch": 0.8158866995073891,
+      "grad_norm": 1.1446128297144444,
+      "learning_rate": 2.9949572250355176e-07,
+      "loss": 0.8222,
+      "step": 1325
+    },
+    {
+      "epoch": 0.8189655172413793,
+      "grad_norm": 1.0766532558632549,
+      "learning_rate": 2.8989616411662826e-07,
+      "loss": 0.8482,
+      "step": 1330
+    },
+    {
+      "epoch": 0.8220443349753694,
+      "grad_norm": 1.1192410514240008,
+      "learning_rate": 2.8043648642303716e-07,
+      "loss": 0.8318,
+      "step": 1335
+    },
+    {
+      "epoch": 0.8251231527093597,
+      "grad_norm": 1.1732228202159805,
+      "learning_rate": 2.7111778290437465e-07,
+      "loss": 0.8542,
+      "step": 1340
+    },
+    {
+      "epoch": 0.8282019704433498,
+      "grad_norm": 1.1091016139195242,
+      "learning_rate": 2.619411307464747e-07,
+      "loss": 0.8419,
+      "step": 1345
+    },
+    {
+      "epoch": 0.8312807881773399,
+      "grad_norm": 1.1364729785535619,
+      "learning_rate": 2.529075907148916e-07,
+      "loss": 0.8515,
+      "step": 1350
+    },
+    {
+      "epoch": 0.8343596059113301,
+      "grad_norm": 1.125887421535199,
+      "learning_rate": 2.440182070322818e-07,
+      "loss": 0.8437,
+      "step": 1355
+    },
+    {
+      "epoch": 0.8374384236453202,
+      "grad_norm": 1.2017918211154734,
+      "learning_rate": 2.352740072577002e-07,
+      "loss": 0.838,
+      "step": 1360
+    },
+    {
+      "epoch": 0.8405172413793104,
+      "grad_norm": 1.12027756687418,
+      "learning_rate": 2.2667600216781757e-07,
+      "loss": 0.8288,
+      "step": 1365
+    },
+    {
+      "epoch": 0.8435960591133005,
+      "grad_norm": 1.2219994394309295,
+      "learning_rate": 2.182251856400826e-07,
+      "loss": 0.8414,
+      "step": 1370
+    },
+    {
+      "epoch": 0.8466748768472906,
+      "grad_norm": 1.2211278136051131,
+      "learning_rate": 2.0992253453783595e-07,
+      "loss": 0.8652,
+      "step": 1375
+    },
+    {
+      "epoch": 0.8497536945812808,
+      "grad_norm": 1.1587232824047158,
+      "learning_rate": 2.0176900859738906e-07,
+      "loss": 0.8524,
+      "step": 1380
+    },
+    {
+      "epoch": 0.8528325123152709,
+      "grad_norm": 1.1859345849325216,
+      "learning_rate": 1.93765550317087e-07,
+      "loss": 0.8285,
+      "step": 1385
+    },
+    {
+      "epoch": 0.8559113300492611,
+      "grad_norm": 1.1246250011569645,
+      "learning_rate": 1.8591308484835833e-07,
+      "loss": 0.8695,
+      "step": 1390
+    },
+    {
+      "epoch": 0.8589901477832512,
+      "grad_norm": 1.0975724898114476,
+      "learning_rate": 1.7821251988877556e-07,
+      "loss": 0.8475,
+      "step": 1395
+    },
+    {
+      "epoch": 0.8620689655172413,
+      "grad_norm": 1.184977416580749,
+      "learning_rate": 1.706647455771302e-07,
+      "loss": 0.8499,
+      "step": 1400
+    },
+    {
+      "epoch": 0.8651477832512315,
+      "grad_norm": 1.1405121835833012,
+      "learning_rate": 1.6327063439053675e-07,
+      "loss": 0.8525,
+      "step": 1405
+    },
+    {
+      "epoch": 0.8682266009852216,
+      "grad_norm": 1.1347635636041886,
+      "learning_rate": 1.56031041043582e-07,
+      "loss": 0.855,
+      "step": 1410
+    },
+    {
+      "epoch": 0.8713054187192119,
+      "grad_norm": 1.171078921940818,
+      "learning_rate": 1.4894680238952318e-07,
+      "loss": 0.8597,
+      "step": 1415
+    },
+    {
+      "epoch": 0.874384236453202,
+      "grad_norm": 1.138783833003801,
+      "learning_rate": 1.4201873732355343e-07,
+      "loss": 0.8481,
+      "step": 1420
+    },
+    {
+      "epoch": 0.8774630541871922,
+      "grad_norm": 1.092525038483926,
+      "learning_rate": 1.3524764668814305e-07,
+      "loss": 0.8605,
+      "step": 1425
+    },
+    {
+      "epoch": 0.8805418719211823,
+      "grad_norm": 1.222379244627012,
+      "learning_rate": 1.2863431318046615e-07,
+      "loss": 0.8796,
+      "step": 1430
+    },
+    {
+      "epoch": 0.8836206896551724,
+      "grad_norm": 1.1249070173434348,
+      "learning_rate": 1.2217950126192523e-07,
+      "loss": 0.8502,
+      "step": 1435
+    },
+    {
+      "epoch": 0.8866995073891626,
+      "grad_norm": 1.1120883579977683,
+      "learning_rate": 1.158839570697861e-07,
+      "loss": 0.8556,
+      "step": 1440
+    },
+    {
+      "epoch": 0.8897783251231527,
+      "grad_norm": 1.0996393014161838,
+      "learning_rate": 1.0974840833092603e-07,
+      "loss": 0.8269,
+      "step": 1445
+    },
+    {
+      "epoch": 0.8928571428571429,
+      "grad_norm": 1.1847229887436155,
+      "learning_rate": 1.0377356427771567e-07,
+      "loss": 0.8519,
+      "step": 1450
+    },
+    {
+      "epoch": 0.895935960591133,
+      "grad_norm": 1.1651787862910556,
+      "learning_rate": 9.796011556603418e-08,
+      "loss": 0.8332,
+      "step": 1455
+    },
+    {
+      "epoch": 0.8990147783251231,
+      "grad_norm": 1.1769494713452262,
+      "learning_rate": 9.230873419543373e-08,
+      "loss": 0.8413,
+      "step": 1460
+    },
+    {
+      "epoch": 0.9020935960591133,
+      "grad_norm": 1.16708817345428,
+      "learning_rate": 8.682007343146198e-08,
+      "loss": 0.8425,
+      "step": 1465
+    },
+    {
+      "epoch": 0.9051724137931034,
+      "grad_norm": 1.1182857603432075,
+      "learning_rate": 8.14947677301468e-08,
+      "loss": 0.8449,
+      "step": 1470
+    },
+    {
+      "epoch": 0.9082512315270936,
+      "grad_norm": 1.1360907628891421,
+      "learning_rate": 7.633343266465731e-08,
+      "loss": 0.8353,
+      "step": 1475
+    },
+    {
+      "epoch": 0.9113300492610837,
+      "grad_norm": 1.1097313290894126,
+      "learning_rate": 7.133666485414858e-08,
+      "loss": 0.8453,
+      "step": 1480
+    },
+    {
+      "epoch": 0.9144088669950738,
+      "grad_norm": 1.115661417760609,
+      "learning_rate": 6.650504189479417e-08,
+      "loss": 0.8186,
+      "step": 1485
+    },
+    {
+      "epoch": 0.9174876847290641,
+      "grad_norm": 1.1418755526101314,
+      "learning_rate": 6.183912229302135e-08,
+      "loss": 0.8647,
+      "step": 1490
+    },
+    {
+      "epoch": 0.9205665024630542,
+      "grad_norm": 1.1871797599246805,
+      "learning_rate": 5.7339445400949763e-08,
+      "loss": 0.8291,
+      "step": 1495
+    },
+    {
+      "epoch": 0.9236453201970444,
+      "grad_norm": 1.119070658397546,
+      "learning_rate": 5.3006531354045596e-08,
+      "loss": 0.8538,
+      "step": 1500
+    },
+    {
+      "epoch": 0.9267241379310345,
+      "grad_norm": 1.092164873747533,
+      "learning_rate": 4.8840881010998536e-08,
+      "loss": 0.8358,
+      "step": 1505
+    },
+    {
+      "epoch": 0.9298029556650246,
+      "grad_norm": 1.0810172270277698,
+      "learning_rate": 4.4842975895823926e-08,
+      "loss": 0.8649,
+      "step": 1510
+    },
+    {
+      "epoch": 0.9328817733990148,
+      "grad_norm": 1.153662401319179,
+      "learning_rate": 4.1013278142201904e-08,
+      "loss": 0.8491,
+      "step": 1515
+    },
+    {
+      "epoch": 0.9359605911330049,
+      "grad_norm": 1.1197006075972242,
+      "learning_rate": 3.7352230440058534e-08,
+      "loss": 0.854,
+      "step": 1520
+    },
+    {
+      "epoch": 0.9390394088669951,
+      "grad_norm": 1.1053995012186697,
+      "learning_rate": 3.386025598439185e-08,
+      "loss": 0.8507,
+      "step": 1525
+    },
+    {
+      "epoch": 0.9421182266009852,
+      "grad_norm": 1.123006710198896,
+      "learning_rate": 3.053775842635453e-08,
+      "loss": 0.8506,
+      "step": 1530
+    },
+    {
+      "epoch": 0.9451970443349754,
+      "grad_norm": 1.1452711767185195,
+      "learning_rate": 2.7385121826593363e-08,
+      "loss": 0.8413,
+      "step": 1535
+    },
+    {
+      "epoch": 0.9482758620689655,
+      "grad_norm": 1.1772793756681545,
+      "learning_rate": 2.4402710610854582e-08,
+      "loss": 0.834,
+      "step": 1540
+    },
+    {
+      "epoch": 0.9513546798029556,
+      "grad_norm": 1.1127272635568173,
+      "learning_rate": 2.159086952785827e-08,
+      "loss": 0.828,
+      "step": 1545
+    },
+    {
+      "epoch": 0.9544334975369458,
+      "grad_norm": 1.1378103593782525,
+      "learning_rate": 1.894992360944786e-08,
+      "loss": 0.8322,
+      "step": 1550
+    },
+    {
+      "epoch": 0.9575123152709359,
+      "grad_norm": 1.0873657483809553,
+      "learning_rate": 1.648017813301739e-08,
+      "loss": 0.8425,
+      "step": 1555
+    },
+    {
+      "epoch": 0.9605911330049262,
+      "grad_norm": 1.1154710130592655,
+      "learning_rate": 1.4181918586225029e-08,
+      "loss": 0.8427,
+      "step": 1560
+    },
+    {
+      "epoch": 0.9636699507389163,
+      "grad_norm": 1.055180361167347,
+      "learning_rate": 1.2055410633990515e-08,
+      "loss": 0.8535,
+      "step": 1565
+    },
+    {
+      "epoch": 0.9667487684729064,
+      "grad_norm": 1.1472755675848318,
+      "learning_rate": 1.0100900087787357e-08,
+      "loss": 0.8382,
+      "step": 1570
+    },
+    {
+      "epoch": 0.9698275862068966,
+      "grad_norm": 1.0297688627896466,
+      "learning_rate": 8.318612877227571e-09,
+      "loss": 0.8586,
+      "step": 1575
+    },
+    {
+      "epoch": 0.9729064039408867,
+      "grad_norm": 1.1452108813379358,
+      "learning_rate": 6.708755023946245e-09,
+      "loss": 0.8447,
+      "step": 1580
+    },
+    {
+      "epoch": 0.9759852216748769,
+      "grad_norm": 1.1529149459273056,
+      "learning_rate": 5.271512617786078e-09,
+      "loss": 0.8572,
+      "step": 1585
+    },
+    {
+      "epoch": 0.979064039408867,
+      "grad_norm": 1.120937946510953,
+      "learning_rate": 4.007051795287098e-09,
+      "loss": 0.8328,
+      "step": 1590
+    },
+    {
+      "epoch": 0.9821428571428571,
+      "grad_norm": 1.1101614829138096,
+      "learning_rate": 2.9155187204820134e-09,
+      "loss": 0.8511,
+      "step": 1595
+    },
+    {
+      "epoch": 0.9852216748768473,
+      "grad_norm": 1.1128980178537908,
+      "learning_rate": 1.997039568000403e-09,
+      "loss": 0.8128,
+      "step": 1600
+    },
+    {
+      "epoch": 0.9883004926108374,
+      "grad_norm": 1.1029990367077254,
+      "learning_rate": 1.2517205084837112e-09,
+      "loss": 0.851,
+      "step": 1605
+    },
+    {
+      "epoch": 0.9913793103448276,
+      "grad_norm": 1.1640517729781794,
+      "learning_rate": 6.796476963130683e-10,
+      "loss": 0.8426,
+      "step": 1610
+    },
+    {
+      "epoch": 0.9944581280788177,
+      "grad_norm": 1.1714192081390722,
+      "learning_rate": 2.8088725964958705e-10,
+      "loss": 0.8611,
+      "step": 1615
+    },
+    {
+      "epoch": 0.9975369458128078,
+      "grad_norm": 1.1569299826810675,
+      "learning_rate": 5.548529279081338e-11,
+      "loss": 0.8289,
+      "step": 1620
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.8464043736457825,
+      "eval_runtime": 739.3506,
+      "eval_samples_per_second": 31.256,
+      "eval_steps_per_second": 0.978,
+      "step": 1624
+    },
+    {
+      "epoch": 1.0,
+      "step": 1624,
+      "total_flos": 1278023857274880.0,
+      "train_loss": 0.8594259261864746,
+      "train_runtime": 26490.5415,
+      "train_samples_per_second": 7.847,
+      "train_steps_per_second": 0.061
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1624,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1278023857274880.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a5b87d37891b65ea2724acba6e92fb6ba51813b3ff2bcba0932525fe643d9bb
+size 6968