Training in progress, step 82, checkpoint

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b7dcbff7de876339f3721ff1c939f730f094a67a65322e589760d84dfc187f2b
+oid sha256:8f0c742b53feb1425b96b726587a249a448321c857059d0326be6a433256989d
 size 243337876

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a43cb4a5f4de31fc38c3a4ce7b6f7228f6262563ae8d783571b380bd5ddf2c4f
+oid sha256:927e1aceb10ca8a5a09ba9f3b979e702d01a5c3186bc4117949ca1004adda2d4
 size 15984

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:11e2b1cffb4921bdba99192dacc5d77c58a792c39665806bd42a62948e383653
+oid sha256:e0e1f75036170375118e5f9d87be75725e02dd06f618342e663c3a2fcf1c9490
 size 15984

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b150ce7d938dadfe982a15cec704a4334028ee41a456c19376fc140abeb0d19a
+oid sha256:391d9702cc8dc33785afdf79b6056fd81880c3ab9a91fed60426027db93b4dad
 size 15984

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c71a7c06a201da5306830f0e5fa81b34fe75a4bbee63aba84548ce7a2c69f8ff
+oid sha256:bc91c10769bb625ec454da9b362de77162b71ac73f8ea218a6e85c620e7eafd2
 size 15984

last-checkpoint/rng_state_4.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3e4f8c3505b514f5f16e9d8b9ea05d7821a43bff0d5492e0525e0d00328848b2
+oid sha256:7a5da09be328273978676931d6dbaed12f8ff8969594968f5df4e874eeca6695
 size 15984

last-checkpoint/rng_state_5.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d6e39a3e9fba990bb9deef49a1169cbc5735add88b55b64d2abe086f8e1a69d1
+oid sha256:b5df7d48106b53066e0ac987347a5bc1382a2920c92bbfa96a6c3cc231a47088
 size 15984

last-checkpoint/rng_state_6.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e9b52f186b357bc5036b0858548d26c95100387477cce1952b774f140db6a115
+oid sha256:3e858e2575fb1c1bddf0a493c421ea145deac4fcdaadd0715ca95e7386dd1e21
 size 15984

last-checkpoint/rng_state_7.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:22ed9a71b7f7cf5460f7cac683afb9a5194e72e569d49ef45eaa9d512779f86b
+oid sha256:6dbcd389f89b3545848d97a92c7b4fc438105e43de3c120941ae6225093e1592
 size 15984

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8965b7671e0740b8770286326f41d265c475f3f4120a2922de6ca2230c871332
+oid sha256:c417616029d0509ec477785aaf653c82e6acacdaba0d839d484ba806f891eb07
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 13.72530460357666,
   "best_model_checkpoint": "miner_id_24/checkpoint-50",
-  "epoch": 1.834862385321101,
+  "epoch": 3.0091743119266057,
   "eval_steps": 50,
-  "global_step": 50,
+  "global_step": 82,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -373,6 +373,230 @@
       "eval_samples_per_second": 46.515,
       "eval_steps_per_second": 1.517,
       "step": 50
+    },
+    {
+      "epoch": 1.8715596330275228,
+      "grad_norm": Infinity,
+      "learning_rate": 4.466294200552814e-05,
+      "loss": 19.4537,
+      "step": 51
+    },
+    {
+      "epoch": 1.908256880733945,
+      "grad_norm": Infinity,
+      "learning_rate": 4.224731442915631e-05,
+      "loss": 14.3513,
+      "step": 52
+    },
+    {
+      "epoch": 1.9449541284403669,
+      "grad_norm": Infinity,
+      "learning_rate": 3.985976942825644e-05,
+      "loss": 15.6143,
+      "step": 53
+    },
+    {
+      "epoch": 1.981651376146789,
+      "grad_norm": Infinity,
+      "learning_rate": 3.750485183043689e-05,
+      "loss": 10.3784,
+      "step": 54
+    },
+    {
+      "epoch": 2.018348623853211,
+      "grad_norm": Infinity,
+      "learning_rate": 3.518704435518989e-05,
+      "loss": 27.644,
+      "step": 55
+    },
+    {
+      "epoch": 2.055045871559633,
+      "grad_norm": Infinity,
+      "learning_rate": 3.291075908078014e-05,
+      "loss": 13.1336,
+      "step": 56
+    },
+    {
+      "epoch": 2.091743119266055,
+      "grad_norm": Infinity,
+      "learning_rate": 3.068032904560308e-05,
+      "loss": 11.2543,
+      "step": 57
+    },
+    {
+      "epoch": 2.128440366972477,
+      "grad_norm": Infinity,
+      "learning_rate": 2.8500000000000015e-05,
+      "loss": 14.9241,
+      "step": 58
+    },
+    {
+      "epoch": 2.165137614678899,
+      "grad_norm": Infinity,
+      "learning_rate": 2.6373922324231034e-05,
+      "loss": 13.203,
+      "step": 59
+    },
+    {
+      "epoch": 2.2018348623853212,
+      "grad_norm": Infinity,
+      "learning_rate": 2.430614312799039e-05,
+      "loss": 13.2085,
+      "step": 60
+    },
+    {
+      "epoch": 2.238532110091743,
+      "grad_norm": Infinity,
+      "learning_rate": 2.2300598546502922e-05,
+      "loss": 14.6191,
+      "step": 61
+    },
+    {
+      "epoch": 2.2752293577981653,
+      "grad_norm": Infinity,
+      "learning_rate": 2.036110624786726e-05,
+      "loss": 15.2782,
+      "step": 62
+    },
+    {
+      "epoch": 2.311926605504587,
+      "grad_norm": Infinity,
+      "learning_rate": 1.8491358165907367e-05,
+      "loss": 10.9829,
+      "step": 63
+    },
+    {
+      "epoch": 2.3486238532110093,
+      "grad_norm": Infinity,
+      "learning_rate": 1.6694913472366796e-05,
+      "loss": 13.5052,
+      "step": 64
+    },
+    {
+      "epoch": 2.385321100917431,
+      "grad_norm": Infinity,
+      "learning_rate": 1.4975191801822934e-05,
+      "loss": 13.2595,
+      "step": 65
+    },
+    {
+      "epoch": 2.4220183486238533,
+      "grad_norm": Infinity,
+      "learning_rate": 1.333546674221826e-05,
+      "loss": 16.4492,
+      "step": 66
+    },
+    {
+      "epoch": 2.458715596330275,
+      "grad_norm": Infinity,
+      "learning_rate": 1.1778859603399603e-05,
+      "loss": 8.1276,
+      "step": 67
+    },
+    {
+      "epoch": 2.4954128440366974,
+      "grad_norm": Infinity,
+      "learning_rate": 1.0308333475527462e-05,
+      "loss": 20.3549,
+      "step": 68
+    },
+    {
+      "epoch": 2.532110091743119,
+      "grad_norm": Infinity,
+      "learning_rate": 8.926687588665526e-06,
+      "loss": 11.7722,
+      "step": 69
+    },
+    {
+      "epoch": 2.5688073394495414,
+      "grad_norm": Infinity,
+      "learning_rate": 7.636551984286994e-06,
+      "loss": 12.8754,
+      "step": 70
+    },
+    {
+      "epoch": 2.6055045871559632,
+      "grad_norm": Infinity,
+      "learning_rate": 6.440382508841363e-06,
+      "loss": 8.2131,
+      "step": 71
+    },
+    {
+      "epoch": 2.6422018348623855,
+      "grad_norm": Infinity,
+      "learning_rate": 5.340456138910954e-06,
+      "loss": 20.0128,
+      "step": 72
+    },
+    {
+      "epoch": 2.6788990825688073,
+      "grad_norm": Infinity,
+      "learning_rate": 4.3388666468566565e-06,
+      "loss": 10.1762,
+      "step": 73
+    },
+    {
+      "epoch": 2.7155963302752295,
+      "grad_norm": Infinity,
+      "learning_rate": 3.437520615203226e-06,
+      "loss": 19.0314,
+      "step": 74
+    },
+    {
+      "epoch": 2.7522935779816513,
+      "grad_norm": Infinity,
+      "learning_rate": 2.638133807351071e-06,
+      "loss": 12.0294,
+      "step": 75
+    },
+    {
+      "epoch": 2.7889908256880735,
+      "grad_norm": Infinity,
+      "learning_rate": 1.942227901523113e-06,
+      "loss": 11.3231,
+      "step": 76
+    },
+    {
+      "epoch": 2.8256880733944953,
+      "grad_norm": Infinity,
+      "learning_rate": 1.3511275941637984e-06,
+      "loss": 13.4429,
+      "step": 77
+    },
+    {
+      "epoch": 2.8623853211009176,
+      "grad_norm": Infinity,
+      "learning_rate": 8.659580783041429e-07,
+      "loss": 15.3239,
+      "step": 78
+    },
+    {
+      "epoch": 2.8990825688073394,
+      "grad_norm": Infinity,
+      "learning_rate": 4.876429016928082e-07,
+      "loss": 12.7188,
+      "step": 79
+    },
+    {
+      "epoch": 2.9357798165137616,
+      "grad_norm": Infinity,
+      "learning_rate": 2.1690220877050392e-07,
+      "loss": 18.9882,
+      "step": 80
+    },
+    {
+      "epoch": 2.9724770642201834,
+      "grad_norm": Infinity,
+      "learning_rate": 5.4251369834105506e-08,
+      "loss": 4.9075,
+      "step": 81
+    },
+    {
+      "epoch": 3.0091743119266057,
+      "grad_norm": Infinity,
+      "learning_rate": 0.0,
+      "loss": 31.5634,
+      "step": 82
     }
   ],
   "logging_steps": 1,
@@ -396,12 +620,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 5.0781150955700224e+17,
+  "total_flos": 8.199767860403241e+17,
   "train_batch_size": 8,
   "trial_name": null,
   "trial_params": null