phi-3.5-new

adapter_config.json

@@ -12,14 +12,14 @@
   "layers_pattern": null,
   "layers_to_transform": null,
   "loftq_config": {},
-  "lora_alpha": 64,
+  "lora_alpha": 32,
   "lora_bias": false,
-  "lora_dropout": 0.1,
+  "lora_dropout": 0.05,
   "megatron_config": null,
   "megatron_core": "megatron.core",
   "modules_to_save": null,
   "peft_type": "LORA",
-  "r": 16,
+  "r": 8,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9530a4f7189fe3a34ddd73af5e9cbbc39d01141ff2a83bc58fc0472af0f0c315
-size 100697728
+oid sha256:263a69bdb9de370405091d8f38a7af188f5c8271576941714e6e32907ca8a968
+size 50365768

all_results.json

@@ -1,12 +1,12 @@
 {
-    "epoch": 9.0,
-    "eval_loss": 1.001703143119812,
-    "eval_runtime": 16.7881,
-    "eval_samples_per_second": 2.561,
-    "eval_steps_per_second": 0.655,
-    "total_flos": 3.918186203657011e+16,
-    "train_loss": 0.8946734860412076,
-    "train_runtime": 2550.8567,
-    "train_samples_per_second": 2.039,
-    "train_steps_per_second": 0.51
+    "epoch": 6.9523809523809526,
+    "eval_loss": 1.0488358736038208,
+    "eval_runtime": 3.908,
+    "eval_samples_per_second": 4.35,
+    "eval_steps_per_second": 2.303,
+    "total_flos": 6666636405768192.0,
+    "train_loss": 1.0272208958455962,
+    "train_runtime": 456.6289,
+    "train_samples_per_second": 1.818,
+    "train_steps_per_second": 0.219
 }

config.json

@@ -132,7 +132,7 @@
   "rope_theta": 10000.0,
   "sliding_window": 262144,
   "tie_word_embeddings": false,
-  "torch_dtype": "bfloat16",
+  "torch_dtype": "float16",
   "transformers_version": "4.46.3",
   "use_cache": false,
   "vocab_size": 32064

eval_results.json

@@ -1,7 +1,7 @@
 {
-    "epoch": 9.0,
-    "eval_loss": 1.001703143119812,
-    "eval_runtime": 16.7881,
-    "eval_samples_per_second": 2.561,
-    "eval_steps_per_second": 0.655
+    "epoch": 6.9523809523809526,
+    "eval_loss": 1.0488358736038208,
+    "eval_runtime": 3.908,
+    "eval_samples_per_second": 4.35,
+    "eval_steps_per_second": 2.303
 }

runs/Jan27_07-19-08_dmlab/events.out.tfevents.1737955149.dmlab.4693.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94d50650fec322040c8739e03f8b8742ee7d1adaa78118a48fc44de863cfb576
+size 10681

runs/Jan27_07-26-28_dmlab/events.out.tfevents.1737955589.dmlab.6099.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9830511c021a38d82724693d0f05c2a2bca3f697ff1ccb6458c4139d8d89f34
+size 11975

runs/Jan27_07-26-28_dmlab/events.out.tfevents.1737956049.dmlab.6099.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a51e8af1c39f169c39c965a77839333888a26ef622142b21e3bdd8f25f1ad6e
+size 354

tokenizer.json

@@ -2,7 +2,7 @@
   "version": "1.0",
   "truncation": {
     "direction": "Right",
-    "max_length": 2048,
+    "max_length": 512,
     "strategy": "LongestFirst",
     "stride": 0
   },

tokenizer_config.json

@@ -121,7 +121,7 @@
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
   "legacy": false,
-  "model_max_length": 4096,
+  "model_max_length": 1024,
   "pad_token": "<unk>",
   "padding_side": "left",
   "sp_model_kwargs": {},

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 9.0,
-    "total_flos": 3.918186203657011e+16,
-    "train_loss": 0.8946734860412076,
-    "train_runtime": 2550.8567,
-    "train_samples_per_second": 2.039,
-    "train_steps_per_second": 0.51
+    "epoch": 6.9523809523809526,
+    "total_flos": 6666636405768192.0,
+    "train_loss": 1.0272208958455962,
+    "train_runtime": 456.6289,
+    "train_samples_per_second": 1.818,
+    "train_steps_per_second": 0.219
 }

trainer_state.json

@@ -1,162 +1,132 @@
 {
-  "best_metric": 0.9914960861206055,
-  "best_model_checkpoint": "/home/labuser/Documents/phi-3/phi-3.5-new/checkpoint-312",
-  "epoch": 9.0,
+  "best_metric": 1.0488358736038208,
+  "best_model_checkpoint": "/home/labuser/Documents/phi-3/phi-3.5-new/checkpoint-42",
+  "epoch": 6.9523809523809526,
   "eval_steps": 500,
-  "global_step": 468,
+  "global_step": 73,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 1.0,
-      "grad_norm": 0.8883442878723145,
-      "learning_rate": 2e-05,
-      "loss": 1.2693,
-      "step": 52
+      "epoch": 0.9523809523809523,
+      "grad_norm": 2.678637981414795,
+      "learning_rate": 0.0001,
+      "loss": 1.5168,
+      "step": 10
     },
     {
-      "epoch": 1.0,
-      "eval_loss": 1.1401562690734863,
-      "eval_runtime": 16.8795,
-      "eval_samples_per_second": 2.547,
-      "eval_steps_per_second": 0.652,
-      "step": 52
+      "epoch": 0.9523809523809523,
+      "eval_loss": 1.3078712224960327,
+      "eval_runtime": 3.9056,
+      "eval_samples_per_second": 4.353,
+      "eval_steps_per_second": 2.304,
+      "step": 10
     },
     {
       "epoch": 2.0,
-      "grad_norm": 0.8219994902610779,
-      "learning_rate": 4e-05,
-      "loss": 0.9808,
-      "step": 104
+      "grad_norm": 2.171729564666748,
+      "learning_rate": 9.635919272833938e-05,
+      "loss": 1.0953,
+      "step": 21
     },
     {
       "epoch": 2.0,
-      "eval_loss": 1.0025322437286377,
-      "eval_runtime": 16.7914,
-      "eval_samples_per_second": 2.561,
-      "eval_steps_per_second": 0.655,
-      "step": 104
+      "eval_loss": 1.1388882398605347,
+      "eval_runtime": 3.9103,
+      "eval_samples_per_second": 4.348,
+      "eval_steps_per_second": 2.302,
+      "step": 21
     },
     {
-      "epoch": 3.0,
-      "grad_norm": 0.8449307680130005,
-      "learning_rate": 6e-05,
-      "loss": 0.895,
-      "step": 156
+      "epoch": 2.9523809523809526,
+      "grad_norm": 2.044914960861206,
+      "learning_rate": 8.715724127386972e-05,
+      "loss": 1.0287,
+      "step": 31
     },
     {
-      "epoch": 3.0,
-      "eval_loss": 1.0123087167739868,
-      "eval_runtime": 16.7952,
-      "eval_samples_per_second": 2.56,
-      "eval_steps_per_second": 0.655,
-      "step": 156
+      "epoch": 2.9523809523809526,
+      "eval_loss": 1.0680148601531982,
+      "eval_runtime": 3.9097,
+      "eval_samples_per_second": 4.348,
+      "eval_steps_per_second": 2.302,
+      "step": 31
     },
     {
       "epoch": 4.0,
-      "grad_norm": 0.622931718826294,
-      "learning_rate": 8e-05,
-      "loss": 0.8651,
-      "step": 208
+      "grad_norm": 3.0813491344451904,
+      "learning_rate": 7.191855733945387e-05,
+      "loss": 0.8858,
+      "step": 42
     },
     {
       "epoch": 4.0,
-      "eval_loss": 1.0015382766723633,
-      "eval_runtime": 16.795,
-      "eval_samples_per_second": 2.56,
-      "eval_steps_per_second": 0.655,
-      "step": 208
+      "eval_loss": 1.0488358736038208,
+      "eval_runtime": 3.9065,
+      "eval_samples_per_second": 4.352,
+      "eval_steps_per_second": 2.304,
+      "step": 42
     },
     {
-      "epoch": 5.0,
-      "grad_norm": 0.6589271426200867,
-      "learning_rate": 0.0001,
-      "loss": 0.8471,
-      "step": 260
+      "epoch": 4.9523809523809526,
+      "grad_norm": 2.399099349975586,
+      "learning_rate": 5.522642316338268e-05,
+      "loss": 0.9442,
+      "step": 52
     },
     {
-      "epoch": 5.0,
-      "eval_loss": 1.0026640892028809,
-      "eval_runtime": 16.7954,
-      "eval_samples_per_second": 2.56,
-      "eval_steps_per_second": 0.655,
-      "step": 260
+      "epoch": 4.9523809523809526,
+      "eval_loss": 1.0551538467407227,
+      "eval_runtime": 3.9027,
+      "eval_samples_per_second": 4.356,
+      "eval_steps_per_second": 2.306,
+      "step": 52
     },
     {
       "epoch": 6.0,
-      "grad_norm": 0.7380354404449463,
-      "learning_rate": 9.938441702975689e-05,
-      "loss": 0.8273,
-      "step": 312
+      "grad_norm": 2.558990240097046,
+      "learning_rate": 3.6218132209150045e-05,
+      "loss": 0.8393,
+      "step": 63
     },
     {
       "epoch": 6.0,
-      "eval_loss": 0.9914960861206055,
-      "eval_runtime": 16.7888,
-      "eval_samples_per_second": 2.561,
-      "eval_steps_per_second": 0.655,
-      "step": 312
-    },
-    {
-      "epoch": 7.0,
-      "grad_norm": 0.7154495716094971,
-      "learning_rate": 9.755282581475769e-05,
-      "loss": 0.8068,
-      "step": 364
-    },
-    {
-      "epoch": 7.0,
-      "eval_loss": 0.9957849979400635,
-      "eval_runtime": 16.7927,
-      "eval_samples_per_second": 2.561,
-      "eval_steps_per_second": 0.655,
-      "step": 364
-    },
-    {
-      "epoch": 8.0,
-      "grad_norm": 0.9513155817985535,
-      "learning_rate": 9.45503262094184e-05,
-      "loss": 0.7883,
-      "step": 416
-    },
-    {
-      "epoch": 8.0,
-      "eval_loss": 1.0050514936447144,
-      "eval_runtime": 16.8033,
-      "eval_samples_per_second": 2.559,
-      "eval_steps_per_second": 0.655,
-      "step": 416
-    },
-    {
-      "epoch": 9.0,
-      "grad_norm": 0.9450660347938538,
-      "learning_rate": 9.045084971874738e-05,
-      "loss": 0.7722,
-      "step": 468
-    },
-    {
-      "epoch": 9.0,
-      "eval_loss": 1.008681058883667,
-      "eval_runtime": 16.7907,
-      "eval_samples_per_second": 2.561,
-      "eval_steps_per_second": 0.655,
-      "step": 468
-    },
-    {
-      "epoch": 9.0,
-      "step": 468,
-      "total_flos": 3.918186203657011e+16,
-      "train_loss": 0.8946734860412076,
-      "train_runtime": 2550.8567,
-      "train_samples_per_second": 2.039,
-      "train_steps_per_second": 0.51
+      "eval_loss": 1.0510764122009277,
+      "eval_runtime": 3.9324,
+      "eval_samples_per_second": 4.323,
+      "eval_steps_per_second": 2.289,
+      "step": 63
+    },
+    {
+      "epoch": 6.9523809523809526,
+      "grad_norm": 2.0949923992156982,
+      "learning_rate": 2.061073738537635e-05,
+      "loss": 0.9065,
+      "step": 73
+    },
+    {
+      "epoch": 6.9523809523809526,
+      "eval_loss": 1.0621706247329712,
+      "eval_runtime": 3.9116,
+      "eval_samples_per_second": 4.346,
+      "eval_steps_per_second": 2.301,
+      "step": 73
+    },
+    {
+      "epoch": 6.9523809523809526,
+      "step": 73,
+      "total_flos": 6666636405768192.0,
+      "train_loss": 1.0272208958455962,
+      "train_runtime": 456.6289,
+      "train_samples_per_second": 1.818,
+      "train_steps_per_second": 0.219
     }
   ],
   "logging_steps": 500,
-  "max_steps": 1300,
+  "max_steps": 100,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 25,
+  "num_train_epochs": 10,
   "save_steps": 500,
   "stateful_callbacks": {
     "EarlyStoppingCallback": {
@@ -179,8 +149,8 @@
       "attributes": {}
     }
   },
-  "total_flos": 3.918186203657011e+16,
-  "train_batch_size": 4,
+  "total_flos": 6666636405768192.0,
+  "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:128eba89cd71392d5ec29707dedc993445621390f2e66243a915aa50897df7e4
+oid sha256:906c3e67879a4a768ad056051772ba574f2f336ffda087c2d37e302a39428848
 size 5624