End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.79303640127182e-06,
-    "train_runtime": 517.536,
-    "train_samples": 28,
-    "train_samples_per_second": 0.618,
-    "train_steps_per_second": 0.039
+    "train_loss": 4.43296080447908e-06,
+    "train_runtime": 1197.492,
+    "train_samples": 30,
+    "train_samples_per_second": 0.267,
+    "train_steps_per_second": 0.017
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:345ef325dc5c93e8fb5c1b98722962661c1ee93c10aba3dc3dad818a4441a599
+oid sha256:a97242d582bfd066c63772e23e7a0da735da1df5aa50251cc2d4c7200274895f
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.79303640127182e-06,
-    "train_runtime": 517.536,
-    "train_samples": 28,
-    "train_samples_per_second": 0.618,
-    "train_steps_per_second": 0.039
+    "train_loss": 4.43296080447908e-06,
+    "train_runtime": 1197.492,
+    "train_samples": 30,
+    "train_samples_per_second": 0.267,
+    "train_steps_per_second": 0.017
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 304.9375,
-      "epoch": 0.5714285714285714,
-      "grad_norm": 4.851372718811035,
+      "completion_length": 376.6875,
+      "epoch": 0.5333333333333333,
+      "grad_norm": 10.583303451538086,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": 0.0,
-      "reward": 2.13264861702919,
-      "reward_std": 1.6104913679882884,
-      "rewards/concensus_correctness_reward_func": 0.7062499988824129,
-      "rewards/consensus_reward_func": 0.125,
+      "loss": -0.0,
+      "reward": 0.8002214969601482,
+      "reward_std": 0.6283993462056969,
+      "rewards/concensus_correctness_reward_func": 0.08431249856948853,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.609211056958884,
+      "rewards/question_recreation_reward_func": 0.302346504497109,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6296875011175871,
+      "rewards/xmlcount_reward_func": 0.3510625039925799,
       "step": 2
     },
     {
-      "completion_length": 354.5416666666667,
+      "completion_length": 338.14285714285717,
       "epoch": 1.0,
-      "grad_norm": 3.619493246078491,
-      "kl": 0.001191105797867446,
+      "grad_norm": 43.01472091674805,
+      "kl": 0.0012165878101118974,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 1.3468344161907833,
-      "reward_std": 0.8530389812464515,
+      "reward": 0.08046834037772246,
+      "reward_std": 0.8427871197595128,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.6258760864535967,
+      "rewards/final_correctness_reward_func": 0.07142857142857142,
+      "rewards/question_recreation_reward_func": 0.20571835126195634,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.47095833470424014,
+      "rewards/xmlcount_reward_func": -0.1966785793857915,
       "step": 4
     },
     {
-      "completion_length": 378.5,
-      "epoch": 1.5714285714285714,
-      "grad_norm": 5.749740123748779,
-      "kl": 0.001892937139928108,
+      "completion_length": 333.21875,
+      "epoch": 1.5333333333333332,
+      "grad_norm": 6.38524866104126,
+      "kl": 0.0012540667476059753,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 1.4423968009650707,
-      "reward_std": 0.9854197157546878,
-      "rewards/concensus_correctness_reward_func": 0.21975000016391277,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 0.4593001059256494,
+      "reward_std": 0.5724506725964602,
+      "rewards/concensus_correctness_reward_func": 0.017249999567866325,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.5246780635789037,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.3556751172291115,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.38546874653548,
+      "rewards/xmlcount_reward_func": 0.08637499739415944,
       "step": 6
     },
     {
-      "completion_length": 356.375,
+      "completion_length": 398.0,
       "epoch": 2.0,
-      "grad_norm": 3.2181859016418457,
-      "kl": 0.002511197264539078,
+      "grad_norm": 22.17148780822754,
+      "kl": 0.0023290804986442837,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 1.5466339296350877,
-      "reward_std": 0.7741681225597858,
-      "rewards/concensus_correctness_reward_func": 0.09300000220537186,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 0.6887792037533862,
+      "reward_std": 0.5893596733554399,
+      "rewards/concensus_correctness_reward_func": 0.06857142703873771,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.521717314918836,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.3127077682875097,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5152499973773956,
+      "rewards/xmlcount_reward_func": 0.3074999994465283,
       "step": 8
     },
     {
-      "completion_length": 325.34375,
-      "epoch": 2.571428571428571,
-      "grad_norm": 3.9849398136138916,
-      "kl": 0.00328226997226011,
+      "completion_length": 302.0,
+      "epoch": 2.533333333333333,
+      "grad_norm": 14.863710403442383,
+      "kl": 0.0018857058057619724,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 1.6753397602587938,
-      "reward_std": 0.9200851991772652,
-      "rewards/concensus_correctness_reward_func": 0.22350000217556953,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 0.3950591804459691,
+      "reward_std": 0.5421361509943381,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5828085069078952,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.22009042837453308,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6190312504768372,
+      "rewards/xmlcount_reward_func": 0.11246875440701842,
       "step": 10
     },
     {
-      "completion_length": 264.2916666666667,
+      "completion_length": 458.35714285714283,
       "epoch": 3.0,
-      "grad_norm": 6.931037902832031,
-      "kl": 0.004592904859843354,
+      "grad_norm": 4.770223617553711,
+      "kl": 0.0021574341891599552,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 2.8149427200357118,
-      "reward_std": 1.5973657455760986,
-      "rewards/concensus_correctness_reward_func": 0.96574999888738,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 0.47677545465662013,
+      "reward_std": 0.8916666923656261,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.07142857142857142,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.6566510275006294,
+      "rewards/final_correctness_reward_func": 0.07142857142857142,
+      "rewards/question_recreation_reward_func": 0.21166830300353467,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6925416647767028,
+      "rewards/strict_format_reward_func": 0.017857142857142856,
+      "rewards/xmlcount_reward_func": 0.10439286141523293,
       "step": 12
     },
     {
-      "completion_length": 281.21875,
-      "epoch": 3.571428571428571,
-      "grad_norm": 5.207312107086182,
-      "kl": 0.005684145929990336,
+      "completion_length": 405.65625,
+      "epoch": 3.533333333333333,
+      "grad_norm": 6.937699794769287,
+      "kl": 0.0019864821624651086,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 1.407125448808074,
-      "reward_std": 0.872270149353426,
-      "rewards/concensus_correctness_reward_func": 0.08893750049173832,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 0.5633653332479298,
+      "reward_std": 0.5968317738734186,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.4857504405081272,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.3372403406538069,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5199375026859343,
+      "rewards/xmlcount_reward_func": 0.16362500004470348,
       "step": 14
     },
     {
-      "completion_length": 349.8333333333333,
+      "completion_length": 403.89285714285717,
       "epoch": 4.0,
-      "grad_norm": 3.5205085277557373,
-      "kl": 0.003587796246089662,
+      "grad_norm": 7.561593055725098,
+      "kl": 0.0025976902896737947,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 1.5020376990238826,
-      "reward_std": 0.8826297484338284,
-      "rewards/concensus_correctness_reward_func": 0.14366666972637177,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 0.6218146264020886,
+      "reward_std": 0.6445266314216757,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5176209922259053,
+      "rewards/final_correctness_reward_func": 0.14285714285714285,
+      "rewards/question_recreation_reward_func": 0.3099217688265656,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5907500150303046,
+      "rewards/xmlcount_reward_func": 0.16903571052742855,
       "step": 16
     },
     {
-      "completion_length": 342.0625,
-      "epoch": 4.571428571428571,
-      "grad_norm": 5.766377925872803,
-      "kl": 0.004773544933414087,
+      "completion_length": 367.0,
+      "epoch": 4.533333333333333,
+      "grad_norm": 23.471553802490234,
+      "kl": 0.02859952425933443,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 2.0188530012965202,
-      "reward_std": 1.9548688947688788,
-      "rewards/concensus_correctness_reward_func": 0.7926874980330467,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 0.31286027817986906,
+      "reward_std": 0.983204610645771,
+      "rewards/concensus_correctness_reward_func": 0.12087499722838402,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.45404057181440294,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.2401727719407063,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.4596250094473362,
+      "rewards/xmlcount_reward_func": -0.04818749101832509,
       "step": 18
     },
     {
-      "completion_length": 348.625,
+      "completion_length": 256.67857142857144,
       "epoch": 5.0,
-      "grad_norm": 3.9206178188323975,
-      "kl": 0.0045129665910887224,
+      "grad_norm": 5.671968460083008,
+      "kl": 0.0041765897892349,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 1.3888407597939174,
-      "reward_std": 1.1015281303940962,
-      "rewards/concensus_correctness_reward_func": 0.14316666622956595,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 0.8634014345173325,
+      "reward_std": 0.8945413868475173,
+      "rewards/concensus_correctness_reward_func": 0.06828571217400688,
+      "rewards/consensus_reward_func": 0.07142857142857142,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.39729905237133306,
+      "rewards/final_correctness_reward_func": 0.21428571428571427,
+      "rewards/question_recreation_reward_func": 0.38347286571349415,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5983750075101852,
+      "rewards/xmlcount_reward_func": 0.12592857437474386,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 2.79303640127182e-06,
-      "train_runtime": 517.536,
-      "train_samples_per_second": 0.618,
-      "train_steps_per_second": 0.039
+      "train_loss": 4.43296080447908e-06,
+      "train_runtime": 1197.492,
+      "train_samples_per_second": 0.267,
+      "train_steps_per_second": 0.017
     }
   ],
   "logging_steps": 2,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a0f1d2a84cc1df784a24da2fa2e9da59d5bf766d339173e6559568855437b41e
+oid sha256:0b9db8a4566df448da6866e240890541215a82d71c06afab2f419ecb57cc2172
 size 5944