Training in progress, step 22000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ba9d75a78fad20f4b1e389f6c85dda0f453be86d800ed2eba32953160cc02033
+oid sha256:f3ed6ac9ccb5e7d64d2e1c9f2879aedec4abdbea9a50739e1287448962b822ce
 size 373077376

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:df2f641838670afd6d1bb0181e8efde74cebba7ddaeaad933397844d1eb9afb6
+oid sha256:21aacf2f2f02d684309501173ad828552422076a0c16aea88131afc48854c0e7
 size 209816139

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eda74d083cd5d9b07d403914b5a235c44dd87bc93a29636e940f36b95f8743f9
+oid sha256:b324ca901edaaaa5841347eeb4f75e6d12219bec163c3b83caffcbf6520d58a3
 size 14917

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:91f1feed6ec98326449107f6ac06aad035f8176b90aa697c6edf6a509039a50c
+oid sha256:f93a058689b8ada6622702c5a5833d4e962616a54302ee183cbf481797944cfb
 size 14917

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c6645e7dc37725bbae83eaf70fb81001a75be54d9a6554f43743dfb20cfc0984
+oid sha256:a6cc8155f9a668ccc650b25a3629943fe3a02ee796145b1ad7dac78628d32ab5
 size 1401

last-checkpoint/trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": 2000,
   "best_metric": 9.218317031860352,
   "best_model_checkpoint": "./artifacts/models/base-250725-test/checkpoint-2000",
-  "epoch": 0.06558523894888724,
+  "epoch": 0.06870834556550091,
   "eval_steps": 1000,
-  "global_step": 21000,
+  "global_step": 22000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -6056,6 +6056,294 @@
       "eval_samples_per_second": 50.848,
       "eval_steps_per_second": 3.186,
       "step": 21000
+    },
+    {
+      "epoch": 0.06566331661430258,
+      "grad_norm": 37.0,
+      "learning_rate": 0.0009993297472148076,
+      "loss": 33.5467,
+      "step": 21025
+    },
+    {
+      "epoch": 0.06574139427971792,
+      "grad_norm": 38.5,
+      "learning_rate": 0.0009993230482511295,
+      "loss": 33.6705,
+      "step": 21050
+    },
+    {
+      "epoch": 0.06581947194513327,
+      "grad_norm": 39.0,
+      "learning_rate": 0.0009993163159993798,
+      "loss": 33.7872,
+      "step": 21075
+    },
+    {
+      "epoch": 0.06589754961054861,
+      "grad_norm": 45.5,
+      "learning_rate": 0.0009993095504600067,
+      "loss": 33.6316,
+      "step": 21100
+    },
+    {
+      "epoch": 0.06597562727596394,
+      "grad_norm": 38.0,
+      "learning_rate": 0.0009993027516334617,
+      "loss": 33.8796,
+      "step": 21125
+    },
+    {
+      "epoch": 0.06605370494137928,
+      "grad_norm": 43.75,
+      "learning_rate": 0.000999295919520198,
+      "loss": 34.0526,
+      "step": 21150
+    },
+    {
+      "epoch": 0.06613178260679463,
+      "grad_norm": 36.0,
+      "learning_rate": 0.000999289054120671,
+      "loss": 34.1438,
+      "step": 21175
+    },
+    {
+      "epoch": 0.06620986027220997,
+      "grad_norm": 38.0,
+      "learning_rate": 0.0009992821554353382,
+      "loss": 33.7974,
+      "step": 21200
+    },
+    {
+      "epoch": 0.06628793793762532,
+      "grad_norm": 46.0,
+      "learning_rate": 0.00099927522346466,
+      "loss": 33.8107,
+      "step": 21225
+    },
+    {
+      "epoch": 0.06636601560304066,
+      "grad_norm": 45.75,
+      "learning_rate": 0.0009992682582090982,
+      "loss": 33.8952,
+      "step": 21250
+    },
+    {
+      "epoch": 0.066444093268456,
+      "grad_norm": 39.5,
+      "learning_rate": 0.0009992612596691171,
+      "loss": 34.201,
+      "step": 21275
+    },
+    {
+      "epoch": 0.06652217093387135,
+      "grad_norm": 49.25,
+      "learning_rate": 0.0009992542278451832,
+      "loss": 34.2007,
+      "step": 21300
+    },
+    {
+      "epoch": 0.06660024859928668,
+      "grad_norm": 42.0,
+      "learning_rate": 0.0009992471627377657,
+      "loss": 34.3501,
+      "step": 21325
+    },
+    {
+      "epoch": 0.06667832626470202,
+      "grad_norm": 48.75,
+      "learning_rate": 0.0009992400643473354,
+      "loss": 34.4321,
+      "step": 21350
+    },
+    {
+      "epoch": 0.06675640393011736,
+      "grad_norm": 43.25,
+      "learning_rate": 0.0009992329326743653,
+      "loss": 34.638,
+      "step": 21375
+    },
+    {
+      "epoch": 0.06683448159553271,
+      "grad_norm": 41.75,
+      "learning_rate": 0.000999225767719331,
+      "loss": 34.588,
+      "step": 21400
+    },
+    {
+      "epoch": 0.06691255926094805,
+      "grad_norm": 44.5,
+      "learning_rate": 0.0009992185694827102,
+      "loss": 34.7111,
+      "step": 21425
+    },
+    {
+      "epoch": 0.0669906369263634,
+      "grad_norm": 50.5,
+      "learning_rate": 0.0009992113379649829,
+      "loss": 34.7677,
+      "step": 21450
+    },
+    {
+      "epoch": 0.06706871459177874,
+      "grad_norm": 62.0,
+      "learning_rate": 0.000999204073166631,
+      "loss": 35.0234,
+      "step": 21475
+    },
+    {
+      "epoch": 0.06714679225719408,
+      "grad_norm": 48.0,
+      "learning_rate": 0.0009991967750881388,
+      "loss": 35.0909,
+      "step": 21500
+    },
+    {
+      "epoch": 0.06722486992260941,
+      "grad_norm": 49.5,
+      "learning_rate": 0.000999189443729993,
+      "loss": 35.4811,
+      "step": 21525
+    },
+    {
+      "epoch": 0.06730294758802476,
+      "grad_norm": 58.0,
+      "learning_rate": 0.0009991820790926824,
+      "loss": 35.2726,
+      "step": 21550
+    },
+    {
+      "epoch": 0.0673810252534401,
+      "grad_norm": 55.5,
+      "learning_rate": 0.0009991746811766975,
+      "loss": 35.629,
+      "step": 21575
+    },
+    {
+      "epoch": 0.06745910291885544,
+      "grad_norm": 44.0,
+      "learning_rate": 0.000999167249982532,
+      "loss": 35.4736,
+      "step": 21600
+    },
+    {
+      "epoch": 0.06753718058427079,
+      "grad_norm": 45.75,
+      "learning_rate": 0.0009991597855106814,
+      "loss": 35.2275,
+      "step": 21625
+    },
+    {
+      "epoch": 0.06761525824968613,
+      "grad_norm": 41.5,
+      "learning_rate": 0.0009991522877616428,
+      "loss": 35.2907,
+      "step": 21650
+    },
+    {
+      "epoch": 0.06769333591510147,
+      "grad_norm": 56.5,
+      "learning_rate": 0.000999144756735916,
+      "loss": 35.2988,
+      "step": 21675
+    },
+    {
+      "epoch": 0.06777141358051682,
+      "grad_norm": 56.0,
+      "learning_rate": 0.000999137192434004,
+      "loss": 35.2948,
+      "step": 21700
+    },
+    {
+      "epoch": 0.06784949124593215,
+      "grad_norm": 42.0,
+      "learning_rate": 0.0009991295948564103,
+      "loss": 35.1186,
+      "step": 21725
+    },
+    {
+      "epoch": 0.06792756891134749,
+      "grad_norm": 43.25,
+      "learning_rate": 0.0009991219640036416,
+      "loss": 35.115,
+      "step": 21750
+    },
+    {
+      "epoch": 0.06800564657676283,
+      "grad_norm": 43.75,
+      "learning_rate": 0.0009991142998762065,
+      "loss": 35.347,
+      "step": 21775
+    },
+    {
+      "epoch": 0.06808372424217818,
+      "grad_norm": 45.0,
+      "learning_rate": 0.000999106602474616,
+      "loss": 35.3008,
+      "step": 21800
+    },
+    {
+      "epoch": 0.06816180190759352,
+      "grad_norm": 66.0,
+      "learning_rate": 0.0009990988717993832,
+      "loss": 35.321,
+      "step": 21825
+    },
+    {
+      "epoch": 0.06823987957300887,
+      "grad_norm": 56.0,
+      "learning_rate": 0.0009990911078510238,
+      "loss": 35.373,
+      "step": 21850
+    },
+    {
+      "epoch": 0.06831795723842421,
+      "grad_norm": 49.25,
+      "learning_rate": 0.000999083310630055,
+      "loss": 35.2404,
+      "step": 21875
+    },
+    {
+      "epoch": 0.06839603490383955,
+      "grad_norm": 46.0,
+      "learning_rate": 0.000999075480136997,
+      "loss": 35.2177,
+      "step": 21900
+    },
+    {
+      "epoch": 0.06847411256925488,
+      "grad_norm": 43.5,
+      "learning_rate": 0.0009990676163723715,
+      "loss": 35.1759,
+      "step": 21925
+    },
+    {
+      "epoch": 0.06855219023467023,
+      "grad_norm": 54.5,
+      "learning_rate": 0.000999059719336703,
+      "loss": 34.7193,
+      "step": 21950
+    },
+    {
+      "epoch": 0.06863026790008557,
+      "grad_norm": 48.25,
+      "learning_rate": 0.0009990517890305175,
+      "loss": 34.6676,
+      "step": 21975
+    },
+    {
+      "epoch": 0.06870834556550091,
+      "grad_norm": 44.75,
+      "learning_rate": 0.0009990438254543442,
+      "loss": 34.4965,
+      "step": 22000
+    },
+    {
+      "epoch": 0.06870834556550091,
+      "eval_loss": 34.531646728515625,
+      "eval_runtime": 102.6371,
+      "eval_samples_per_second": 50.693,
+      "eval_steps_per_second": 3.176,
+      "step": 22000
     }
   ],
   "logging_steps": 25,
@@ -6075,7 +6363,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2.663111367480836e+18,
+  "total_flos": 2.7899608404454277e+18,
   "train_batch_size": 8,
   "trial_name": null,
   "trial_params": null