End of training

README.md

@@ -38,7 +38,7 @@ This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing
 ### Framework versions
 
 - TRL: 0.15.2
-- Transformers: 4.51.1
+- Transformers: 4.51.3
 - Pytorch: 2.5.1
 - Datasets: 3.5.0
 - Tokenizers: 0.21.1

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.9973208736701053e-06,
-    "train_runtime": 1100.2769,
-    "train_samples": 65,
-    "train_samples_per_second": 0.291,
-    "train_steps_per_second": 0.018
+    "train_loss": 1.3347715139389039e-06,
+    "train_runtime": 1407.677,
+    "train_samples": 36,
+    "train_samples_per_second": 0.227,
+    "train_steps_per_second": 0.014
 }

config.json

@@ -21,7 +21,7 @@
   "sliding_window": 32768,
   "tie_word_embeddings": true,
   "torch_dtype": "float32",
-  "transformers_version": "4.51.1",
+  "transformers_version": "4.51.3",
   "use_cache": true,
   "use_sliding_window": false,
   "vocab_size": 151936

generation_config.json

@@ -10,5 +10,5 @@
   "temperature": 0.7,
   "top_k": 20,
   "top_p": 0.8,
-  "transformers_version": "4.51.1"
+  "transformers_version": "4.51.3"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:309c5b799689c04a28d5190c38d3953aaf31c5bb0e994e8139bc49348b954f59
+oid sha256:8a7095eccfab0510358cf3e5961c0d9de9ee2f7e765e4690925765d62c313c01
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.9973208736701053e-06,
-    "train_runtime": 1100.2769,
-    "train_samples": 65,
-    "train_samples_per_second": 0.291,
-    "train_steps_per_second": 0.018
+    "train_loss": 1.3347715139389039e-06,
+    "train_runtime": 1407.677,
+    "train_samples": 36,
+    "train_samples_per_second": 0.227,
+    "train_steps_per_second": 0.014
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 2.246153846153846,
+  "epoch": 4.0,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,209 +10,209 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 300.59375,
-      "epoch": 0.24615384615384617,
-      "grad_norm": 18.611722946166992,
+      "completion_length": 334.375,
+      "epoch": 0.4444444444444444,
+      "grad_norm": 9.98232364654541,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": 0.0,
-      "reward": 0.43311248649843037,
-      "reward_std": 0.4912636169756297,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "loss": -0.0,
+      "reward": 0.6942873517982662,
+      "reward_std": 0.7790399911464192,
+      "rewards/concensus_correctness_reward_func": 0.02824999950826168,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.203706243657507,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.3245998634956777,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.16690625017508864,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.2633124995045364,
       "step": 2
     },
     {
-      "completion_length": 282.6875,
-      "epoch": 0.49230769230769234,
-      "grad_norm": 72.78466796875,
-      "kl": 0.0019120956967526581,
+      "completion_length": 393.8125,
+      "epoch": 0.8888888888888888,
+      "grad_norm": 16.888904571533203,
+      "kl": 0.0009467434083489934,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 0.3139393083984032,
-      "reward_std": 0.2587688213970978,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.42340855981456116,
+      "reward_std": 1.1682742208358832,
+      "rewards/concensus_correctness_reward_func": 0.0885000005364418,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.2230018057161942,
+      "rewards/question_recreation_reward_func": 0.34325233817799017,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.09093750081956387,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": -0.1489687574794516,
       "step": 4
     },
     {
-      "completion_length": 289.09375,
-      "epoch": 0.7384615384615385,
-      "grad_norm": 26.63272476196289,
-      "kl": 0.0014575607419828884,
+      "completion_length": 384.6666666666667,
+      "epoch": 1.2222222222222223,
+      "grad_norm": 4.168839454650879,
+      "kl": 0.0010957487247651443,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 0.17699704214464873,
-      "reward_std": 0.5053361464524642,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.42676496505737305,
+      "reward_std": 1.1239696635554235,
+      "rewards/concensus_correctness_reward_func": 0.07158333559830983,
+      "rewards/consensus_reward_func": 0.08333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.11677830037660897,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.2778899338251601,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.06478125229477882,
+      "rewards/xmlcount_reward_func": -0.08937498802940051,
       "step": 6
     },
     {
-      "completion_length": 359.21875,
-      "epoch": 0.9846153846153847,
-      "grad_norm": 13.069327354431152,
-      "kl": 0.0015969037485774606,
+      "completion_length": 377.21875,
+      "epoch": 1.6666666666666665,
+      "grad_norm": 17.552459716796875,
+      "kl": 0.0014122225747996708,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 0.43695820518769324,
-      "reward_std": 0.7170209622272523,
-      "rewards/concensus_correctness_reward_func": 0.02368750050663948,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 1.894684904254973,
+      "reward_std": 0.6818022921843294,
+      "rewards/concensus_correctness_reward_func": 1.3700000010430813,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.20998945622704923,
-      "rewards/soft_format_reward_func": 0.0,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.15087230456992984,
+      "rewards/soft_format_reward_func": 0.03125,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.04671874619089067,
+      "rewards/xmlcount_reward_func": 0.2175624999217689,
       "step": 8
     },
     {
-      "completion_length": 307.22222222222223,
-      "epoch": 1.123076923076923,
-      "grad_norm": 27.43773078918457,
-      "kl": 0.0012352249015950495,
+      "completion_length": 390.4166666666667,
+      "epoch": 2.0,
+      "grad_norm": 4.254613399505615,
+      "kl": 0.0014189543289830908,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 0.25958741394182044,
-      "reward_std": 0.30834134988900685,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.6312103668848673,
+      "reward_std": 0.8441808245455226,
+      "rewards/concensus_correctness_reward_func": 0.03816666702429453,
+      "rewards/consensus_reward_func": 0.08333333333333333,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.23397630225453112,
+      "rewards/question_recreation_reward_func": 0.30179370411982137,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.02561110920376248,
+      "rewards/xmlcount_reward_func": 0.2079166683057944,
       "step": 10
     },
     {
-      "completion_length": 270.84375,
-      "epoch": 1.3692307692307693,
-      "grad_norm": 25.50453758239746,
-      "kl": 0.0019077695997111732,
+      "completion_length": 383.65625,
+      "epoch": 2.4444444444444446,
+      "grad_norm": 5.887304306030273,
+      "kl": 0.001406861141731497,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 0.3587973761605099,
-      "reward_std": 0.3584526478080079,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.550885591423139,
+      "reward_std": 0.9971844495157711,
+      "rewards/concensus_correctness_reward_func": 0.05999999865889549,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.1832348833559081,
+      "rewards/question_recreation_reward_func": 0.27963559550698847,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.11306249815970659,
+      "rewards/xmlcount_reward_func": 0.08625000040046871,
       "step": 12
     },
     {
-      "completion_length": 235.40625,
-      "epoch": 1.6153846153846154,
-      "grad_norm": 32.86248016357422,
-      "kl": 0.007025928747680155,
+      "completion_length": 342.5625,
+      "epoch": 2.888888888888889,
+      "grad_norm": 5.372499942779541,
+      "kl": 0.0016876545487320982,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 0.05113118403824046,
-      "reward_std": 0.3909947890788317,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 1.126185760833323,
+      "reward_std": 1.1402174881659448,
+      "rewards/concensus_correctness_reward_func": 0.17393750324845314,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.1733811855665408,
-      "rewards/soft_format_reward_func": 0.0,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.39993578620487824,
+      "rewards/soft_format_reward_func": 0.015625,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.1222500023432076,
+      "rewards/xmlcount_reward_func": 0.22418749518692493,
       "step": 14
     },
     {
-      "completion_length": 337.96875,
-      "epoch": 1.8615384615384616,
-      "grad_norm": 8.521805763244629,
-      "kl": 0.0017720398755045608,
+      "completion_length": 293.2083333333333,
+      "epoch": 3.2222222222222223,
+      "grad_norm": 32.7645263671875,
+      "kl": 0.0025412932833811888,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 0.5856583423446864,
-      "reward_std": 0.3932318560255226,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.5968316902096072,
+      "reward_std": 0.8732532948100319,
+      "rewards/concensus_correctness_reward_func": 0.0798333336909612,
+      "rewards/consensus_reward_func": 0.08333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.22318959544645622,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.2175400418539842,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.11246874975040555,
+      "rewards/strict_format_reward_func": 0.020833333333333332,
+      "rewards/xmlcount_reward_func": 0.11195833049714565,
       "step": 16
     },
     {
-      "completion_length": 425.8888888888889,
-      "epoch": 2.0,
-      "grad_norm": 2.303975820541382,
-      "kl": 0.001331323615482284,
+      "completion_length": 332.9375,
+      "epoch": 3.6666666666666665,
+      "grad_norm": 29.87108612060547,
+      "kl": 0.0027291466103633866,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 0.29701078445133233,
-      "reward_std": 0.34896431716010234,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 1.1393497553654015,
+      "reward_std": 0.9988422698806971,
+      "rewards/concensus_correctness_reward_func": 0.0886249989271164,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.1177330059516761,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.4011309971101582,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.17927778015534082,
+      "rewards/xmlcount_reward_func": 0.399593748152256,
       "step": 18
     },
     {
-      "completion_length": 228.78125,
-      "epoch": 2.246153846153846,
-      "grad_norm": 18.418474197387695,
-      "kl": 0.0023572802456328645,
+      "completion_length": 395.9583333333333,
+      "epoch": 4.0,
+      "grad_norm": 3.4084742069244385,
+      "kl": 0.0017951912304852158,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 0.29123968281783164,
-      "reward_std": 0.35772098499001004,
+      "reward": 0.6872136739548296,
+      "reward_std": 0.6094399807043374,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.08333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.1626146831549704,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.3321720055149247,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0661249989643693,
+      "rewards/xmlcount_reward_func": 0.18837500115235647,
       "step": 20
     },
     {
-      "epoch": 2.246153846153846,
+      "epoch": 4.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 1.9973208736701053e-06,
-      "train_runtime": 1100.2769,
-      "train_samples_per_second": 0.291,
-      "train_steps_per_second": 0.018
+      "train_loss": 1.3347715139389039e-06,
+      "train_runtime": 1407.677,
+      "train_samples_per_second": 0.227,
+      "train_steps_per_second": 0.014
     }
   ],
   "logging_steps": 2,
   "max_steps": 20,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 3,
+  "num_train_epochs": 5,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5f013d450afc695974c4d4c1ee69138debf6d9d97a4fd6b63c984545f8a1bcfb
+oid sha256:a0f1d2a84cc1df784a24da2fa2e9da59d5bf766d339173e6559568855437b41e
 size 5944