| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 4.285714285714286, | |
| "eval_steps": 500, | |
| "global_step": 30, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "advantages_interactive_phase": -7.450580596923828e-09, | |
| "clip_ratio": 0.009601324272807688, | |
| "clue_civilian_adjusted_reward_mean": 0.04645636805253976, | |
| "clue_civilian_advantage_adjustment": -0.0340991875030158, | |
| "clue_civilian_baseline": 0.0340991875030158, | |
| "clue_civilian_raw_reward_mean": 0.08055555555555557, | |
| "clue_civilian_votes_avg": 0.5833333333333333, | |
| "clue_invalid_votes": 2.875, | |
| "clue_na_votes": 0.375, | |
| "clue_spy_adjusted_reward": -0.13936911300005017, | |
| "clue_spy_advantage_adjustment": 0.10229755366661651, | |
| "clue_spy_baseline": -0.10229755366661651, | |
| "clue_spy_raw_reward": -0.24166666666666672, | |
| "clue_spy_votes_received": 3.0, | |
| "clue_suspicion_potential_psi": 2.4166666666666665, | |
| "clue_total_valid_votes": 4.75, | |
| "completion_length": 468.671875, | |
| "epoch": 0.14285714285714285, | |
| "grad_norm": 1.5805621147155762, | |
| "kl": 0.0014362335205078125, | |
| "learning_rate": 3.5714285714285716e-07, | |
| "loss": 0.0009, | |
| "loss_interactive_phase": 0.0008710725232958794, | |
| "reward": 0.04593749572205778, | |
| "reward_original_clue_max": 0.13422530971612442, | |
| "reward_original_clue_mean": -2.210607728011897e-09, | |
| "reward_original_clue_min": -0.2361905828296588, | |
| "reward_original_clue_std": 0.15096539176990595, | |
| "reward_original_decision_max": 2.1412500000000003, | |
| "reward_original_decision_mean": 0.18375000000000005, | |
| "reward_original_decision_min": -1.5, | |
| "reward_original_decision_std": 1.5282191941515817, | |
| "reward_original_overall_mean": 0.09187499889469616, | |
| "reward_original_overall_std": 1.1872952864471322, | |
| "reward_std": 1.1281701095850887, | |
| "step": 1 | |
| }, | |
| { | |
| "advantages_interactive_phase": 1.1699739843606949e-08, | |
| "clip_ratio": 0.01009189459728077, | |
| "clue_civilian_adjusted_reward_mean": 0.032244541979207814, | |
| "clue_civilian_advantage_adjustment": -0.04969990246523662, | |
| "clue_civilian_baseline": 0.04969990246523663, | |
| "clue_civilian_raw_reward_mean": 0.08194444444444444, | |
| "clue_civilian_votes_avg": 0.29166666666666663, | |
| "clue_invalid_votes": 3.375, | |
| "clue_na_votes": 1.0, | |
| "clue_spy_adjusted_reward": -0.09673363023500019, | |
| "clue_spy_advantage_adjustment": 0.14909970309833318, | |
| "clue_spy_baseline": -0.14909970309833318, | |
| "clue_spy_raw_reward": -0.24583333333333335, | |
| "clue_spy_votes_received": 2.75, | |
| "clue_suspicion_potential_psi": 2.458333333333333, | |
| "clue_total_valid_votes": 3.625, | |
| "completion_length": 346.546875, | |
| "epoch": 0.2857142857142857, | |
| "grad_norm": 0.9062501192092896, | |
| "kl": 0.0015811920166015625, | |
| "learning_rate": 7.142857142857143e-07, | |
| "loss": 0.0006, | |
| "loss_interactive_phase": 0.000601769017521292, | |
| "reward": 0.02179688058128391, | |
| "reward_original_clue_max": 0.12385744329827358, | |
| "reward_original_clue_mean": -1.0743441833859987e-09, | |
| "reward_original_clue_min": -0.2033983952724114, | |
| "reward_original_clue_std": 0.13295147035012433, | |
| "reward_original_decision_max": 1.3687500000000001, | |
| "reward_original_decision_mean": 0.08718750000000001, | |
| "reward_original_decision_min": -1.5, | |
| "reward_original_decision_std": 1.2333472361619537, | |
| "reward_original_overall_mean": 0.04359374946282798, | |
| "reward_original_overall_std": 1.0888861639122744, | |
| "reward_std": 0.9135217889060865, | |
| "step": 2 | |
| }, | |
| { | |
| "advantages_interactive_phase": -3.91155481338501e-08, | |
| "clip_ratio": 0.010360982327256352, | |
| "clue_civilian_adjusted_reward_mean": -0.012963048645367207, | |
| "clue_civilian_advantage_adjustment": -0.06712971531203388, | |
| "clue_civilian_baseline": 0.06712971531203388, | |
| "clue_civilian_raw_reward_mean": 0.05416666666666666, | |
| "clue_civilian_votes_avg": 0.5, | |
| "clue_invalid_votes": 4.125, | |
| "clue_na_votes": 0.25, | |
| "clue_spy_adjusted_reward": 0.03888914454732575, | |
| "clue_spy_advantage_adjustment": 0.20138914454732576, | |
| "clue_spy_baseline": -0.20138914454732576, | |
| "clue_spy_raw_reward": -0.16249999999999998, | |
| "clue_spy_votes_received": 2.125, | |
| "clue_suspicion_potential_psi": 1.6249999999999998, | |
| "clue_total_valid_votes": 3.625, | |
| "completion_length": 469.40625, | |
| "epoch": 0.42857142857142855, | |
| "grad_norm": 1.7866086959838867, | |
| "kl": 0.0020046234130859375, | |
| "learning_rate": 1.0714285714285714e-06, | |
| "loss": 0.001, | |
| "loss_interactive_phase": 0.0009609556873328984, | |
| "reward": -0.07253908214457255, | |
| "reward_original_clue_max": 0.17909297508117578, | |
| "reward_original_clue_mean": -3.4719396715260364e-10, | |
| "reward_original_clue_min": -0.1906668791792172, | |
| "reward_original_clue_std": 0.148119477615202, | |
| "reward_original_decision_max": 0.8700000000000001, | |
| "reward_original_decision_mean": -0.29015625, | |
| "reward_original_decision_min": -1.5, | |
| "reward_original_decision_std": 1.012053904965003, | |
| "reward_original_overall_mean": -0.14507812517359697, | |
| "reward_original_overall_std": 1.0090200646515894, | |
| "reward_std": 1.0390325135884786, | |
| "step": 3 | |
| }, | |
| { | |
| "advantages_interactive_phase": 1.618172973394394e-08, | |
| "clip_ratio": 0.011920451768673956, | |
| "clue_civilian_adjusted_reward_mean": -0.016897605204000857, | |
| "clue_civilian_advantage_adjustment": -0.05578649409288976, | |
| "clue_civilian_baseline": 0.05578649409288975, | |
| "clue_civilian_raw_reward_mean": 0.038888888888888896, | |
| "clue_civilian_votes_avg": 0.5833333333333333, | |
| "clue_invalid_votes": 3.75, | |
| "clue_na_votes": 0.75, | |
| "clue_spy_adjusted_reward": 0.05069280585926233, | |
| "clue_spy_advantage_adjustment": 0.16735947252592903, | |
| "clue_spy_baseline": -0.16735947252592903, | |
| "clue_spy_raw_reward": -0.11666666666666668, | |
| "clue_spy_votes_received": 1.75, | |
| "clue_suspicion_potential_psi": 1.1666666666666665, | |
| "clue_total_valid_votes": 3.5, | |
| "completion_length": 349.9765625, | |
| "epoch": 0.5714285714285714, | |
| "grad_norm": 0.8659637570381165, | |
| "kl": 0.0020847320556640625, | |
| "learning_rate": 1.4285714285714286e-06, | |
| "loss": 0.0006, | |
| "loss_interactive_phase": 0.0005855783383594826, | |
| "reward": -0.09585936751868138, | |
| "reward_original_clue_max": 0.12609553713896215, | |
| "reward_original_clue_mean": -2.438185060966991e-09, | |
| "reward_original_clue_min": -0.1539659667175763, | |
| "reward_original_clue_std": 0.11102248687112745, | |
| "reward_original_decision_max": 1.7175, | |
| "reward_original_decision_mean": -0.3834375, | |
| "reward_original_decision_min": -1.5, | |
| "reward_original_decision_std": 1.318500391473726, | |
| "reward_original_overall_mean": -0.1917187512190925, | |
| "reward_original_overall_std": 1.0483758882247372, | |
| "reward_std": 0.8920968279522142, | |
| "step": 4 | |
| }, | |
| { | |
| "advantages_interactive_phase": -9.313225746154785e-09, | |
| "clip_ratio": 0.011668159277178347, | |
| "clue_civilian_adjusted_reward_mean": 0.012703904980288123, | |
| "clue_civilian_advantage_adjustment": -0.06090720613082299, | |
| "clue_civilian_baseline": 0.060907206130823004, | |
| "clue_civilian_raw_reward_mean": 0.07361111111111111, | |
| "clue_civilian_votes_avg": 0.6666666666666666, | |
| "clue_invalid_votes": 3.125, | |
| "clue_na_votes": 0.0, | |
| "clue_spy_adjusted_reward": -0.038111721525748175, | |
| "clue_spy_advantage_adjustment": 0.18272161180758517, | |
| "clue_spy_baseline": -0.18272161180758517, | |
| "clue_spy_raw_reward": -0.22083333333333333, | |
| "clue_spy_votes_received": 2.875, | |
| "clue_suspicion_potential_psi": 2.208333333333334, | |
| "clue_total_valid_votes": 4.875, | |
| "completion_length": 454.71875, | |
| "epoch": 0.7142857142857143, | |
| "grad_norm": 1.5932543277740479, | |
| "kl": 0.0032215118408203125, | |
| "learning_rate": 1.7857142857142859e-06, | |
| "loss": 0.0011, | |
| "loss_interactive_phase": 0.0011206967756152153, | |
| "reward": 0.020976557431831903, | |
| "reward_original_clue_max": 0.14661833481616776, | |
| "reward_original_clue_mean": -1.646220952741459e-09, | |
| "reward_original_clue_min": -0.24318362303306112, | |
| "reward_original_clue_std": 0.15305170150846256, | |
| "reward_original_decision_max": 1.6425, | |
| "reward_original_decision_mean": 0.08390625000000004, | |
| "reward_original_decision_min": -1.5, | |
| "reward_original_decision_std": 1.4370029970655516, | |
| "reward_original_overall_mean": 0.04195312417688955, | |
| "reward_original_overall_std": 1.1718515588438811, | |
| "reward_std": 1.1204482532340438, | |
| "step": 5 | |
| }, | |
| { | |
| "advantages_interactive_phase": 1.3998942449688911e-08, | |
| "clip_ratio": 0.010447208071127534, | |
| "clue_civilian_adjusted_reward_mean": 0.019708324802807216, | |
| "clue_civilian_advantage_adjustment": -0.06084723075274834, | |
| "clue_civilian_baseline": 0.06084723075274835, | |
| "clue_civilian_raw_reward_mean": 0.08055555555555557, | |
| "clue_civilian_votes_avg": 0.5833333333333333, | |
| "clue_invalid_votes": 1.875, | |
| "clue_na_votes": 1.375, | |
| "clue_spy_adjusted_reward": -0.05912497942333426, | |
| "clue_spy_advantage_adjustment": 0.18254168724333242, | |
| "clue_spy_baseline": -0.18254168724333242, | |
| "clue_spy_raw_reward": -0.2416666666666667, | |
| "clue_spy_votes_received": 3.0, | |
| "clue_suspicion_potential_psi": 2.416666666666667, | |
| "clue_total_valid_votes": 4.75, | |
| "completion_length": 334.2265625, | |
| "epoch": 0.8571428571428571, | |
| "grad_norm": 0.8248350024223328, | |
| "kl": 0.0023393630981445312, | |
| "learning_rate": 2.1428571428571427e-06, | |
| "loss": 0.0006, | |
| "loss_interactive_phase": 0.0005622203752864152, | |
| "reward": 0.0834375066860392, | |
| "reward_original_clue_max": 0.1274395314956929, | |
| "reward_original_clue_mean": -1.253728150484891e-09, | |
| "reward_original_clue_min": -0.188569575974057, | |
| "reward_original_clue_std": 0.12904785603086397, | |
| "reward_original_decision_max": 2.1412500000000003, | |
| "reward_original_decision_mean": 0.33375000000000005, | |
| "reward_original_decision_min": -1.425, | |
| "reward_original_decision_std": 1.405177486682367, | |
| "reward_original_overall_mean": 0.16687499937313596, | |
| "reward_original_overall_std": 1.1144284606169126, | |
| "reward_std": 0.9262521229778956, | |
| "step": 6 | |
| }, | |
| { | |
| "advantages_interactive_phase": 2.421438694000244e-08, | |
| "clip_ratio": 0.011843746062368155, | |
| "clue_civilian_adjusted_reward_mean": 0.06142050400483314, | |
| "clue_civilian_advantage_adjustment": -0.09830171821738909, | |
| "clue_civilian_baseline": 0.0983017182173891, | |
| "clue_civilian_raw_reward_mean": 0.15972222222222224, | |
| "clue_civilian_votes_avg": 0.5833333333333333, | |
| "clue_invalid_votes": 0.625, | |
| "clue_na_votes": 0.25, | |
| "clue_spy_adjusted_reward": -0.18426152068943952, | |
| "clue_spy_advantage_adjustment": 0.2949051459772272, | |
| "clue_spy_baseline": -0.2949051459772272, | |
| "clue_spy_raw_reward": -0.47916666666666674, | |
| "clue_spy_votes_received": 5.375, | |
| "clue_suspicion_potential_psi": 4.791666666666667, | |
| "clue_total_valid_votes": 7.125, | |
| "completion_length": 403.921875, | |
| "epoch": 1.0, | |
| "grad_norm": 1.83256995677948, | |
| "kl": 0.008373260498046875, | |
| "learning_rate": 2.5e-06, | |
| "loss": 0.0082, | |
| "loss_interactive_phase": 0.008202007971704006, | |
| "reward": 0.33738282406500975, | |
| "reward_original_clue_max": 0.16317140174540612, | |
| "reward_original_clue_mean": -2.1687350280430798e-09, | |
| "reward_original_clue_min": -0.2651790850966792, | |
| "reward_original_clue_std": 0.1737400336164854, | |
| "reward_original_decision_max": 2.1412500000000003, | |
| "reward_original_decision_mean": 1.34953125, | |
| "reward_original_decision_min": -1.05, | |
| "reward_original_decision_std": 1.2417606105114989, | |
| "reward_original_overall_mean": 0.6747656239156326, | |
| "reward_original_overall_std": 1.2304992480944565, | |
| "reward_std": 1.149772112760493, | |
| "step": 7 | |
| }, | |
| { | |
| "advantages_interactive_phase": 7.101334631443024e-09, | |
| "clip_ratio": 0.010323125752620399, | |
| "clue_civilian_adjusted_reward_mean": 0.014969077351818358, | |
| "clue_civilian_advantage_adjustment": -0.13919758931484832, | |
| "clue_civilian_baseline": 0.13919758931484832, | |
| "clue_civilian_raw_reward_mean": 0.1541666666666667, | |
| "clue_civilian_votes_avg": 0.6249999999999999, | |
| "clue_invalid_votes": 0.5, | |
| "clue_na_votes": 0.375, | |
| "clue_spy_adjusted_reward": -0.0449072601736762, | |
| "clue_spy_advantage_adjustment": 0.41759273982632383, | |
| "clue_spy_baseline": -0.41759273982632383, | |
| "clue_spy_raw_reward": -0.4625, | |
| "clue_spy_votes_received": 5.25, | |
| "clue_suspicion_potential_psi": 4.625, | |
| "clue_total_valid_votes": 7.125, | |
| "completion_length": 296.3125, | |
| "epoch": 1.1428571428571428, | |
| "grad_norm": 0.8603692054748535, | |
| "kl": 0.00751495361328125, | |
| "learning_rate": 2.8571428571428573e-06, | |
| "loss": 0.0006, | |
| "loss_interactive_phase": 0.0005805297987535596, | |
| "reward": 0.32882812679327855, | |
| "reward_original_clue_max": 0.14714505206208714, | |
| "reward_original_clue_mean": -7.029555287255823e-09, | |
| "reward_original_clue_min": -0.25208323488394496, | |
| "reward_original_clue_std": 0.1584445317000752, | |
| "reward_original_decision_max": 2.49, | |
| "reward_original_decision_mean": 1.3153125, | |
| "reward_original_decision_min": -1.05, | |
| "reward_original_decision_std": 1.3998854722184924, | |
| "reward_original_overall_mean": 0.6576562464852225, | |
| "reward_original_overall_std": 1.2712911404891047, | |
| "reward_std": 1.006177807237865, | |
| "step": 8 | |
| }, | |
| { | |
| "advantages_interactive_phase": 8.754432201385498e-08, | |
| "clip_ratio": 0.00764932727906853, | |
| "clue_civilian_adjusted_reward_mean": -0.04392835440414705, | |
| "clue_civilian_advantage_adjustment": -0.12309502107081373, | |
| "clue_civilian_baseline": 0.12309502107081373, | |
| "clue_civilian_raw_reward_mean": 0.07916666666666668, | |
| "clue_civilian_votes_avg": 1.25, | |
| "clue_invalid_votes": 0.375, | |
| "clue_na_votes": 0.25, | |
| "clue_spy_adjusted_reward": 0.13178503837842287, | |
| "clue_spy_advantage_adjustment": 0.36928503837842286, | |
| "clue_spy_baseline": -0.36928503837842286, | |
| "clue_spy_raw_reward": -0.23750000000000002, | |
| "clue_spy_votes_received": 3.625, | |
| "clue_suspicion_potential_psi": 2.375, | |
| "clue_total_valid_votes": 7.375, | |
| "completion_length": 383.28125, | |
| "epoch": 1.2857142857142856, | |
| "grad_norm": 1.2030161619186401, | |
| "kl": 0.021331787109375, | |
| "learning_rate": 3.2142857142857147e-06, | |
| "loss": 0.0013, | |
| "loss_interactive_phase": 0.0013305692409630865, | |
| "reward": 0.15667972972003488, | |
| "reward_original_clue_max": 0.31233062085321145, | |
| "reward_original_clue_mean": -6.208504575678717e-09, | |
| "reward_original_clue_min": -0.36730352417905254, | |
| "reward_original_clue_std": 0.2591782484917472, | |
| "reward_original_decision_max": 1.3687500000000001, | |
| "reward_original_decision_mean": 0.6267187500000002, | |
| "reward_original_decision_min": -0.7012499999999999, | |
| "reward_original_decision_std": 0.7808322541858488, | |
| "reward_original_overall_mean": 0.31335937189574775, | |
| "reward_original_overall_std": 0.9976451254390301, | |
| "reward_std": 0.8997144106280117, | |
| "step": 9 | |
| }, | |
| { | |
| "advantages_interactive_phase": -5.820766091346741e-10, | |
| "clip_ratio": 0.009187435440253466, | |
| "clue_civilian_adjusted_reward_mean": 0.03822758988085955, | |
| "clue_civilian_advantage_adjustment": -0.11732796567469601, | |
| "clue_civilian_baseline": 0.11732796567469601, | |
| "clue_civilian_raw_reward_mean": 0.15555555555555556, | |
| "clue_civilian_votes_avg": 0.7083333333333333, | |
| "clue_invalid_votes": 0.125, | |
| "clue_na_votes": 0.375, | |
| "clue_spy_adjusted_reward": -0.11468278722393674, | |
| "clue_spy_advantage_adjustment": 0.35198387944273, | |
| "clue_spy_baseline": -0.35198387944273, | |
| "clue_spy_raw_reward": -0.4666666666666667, | |
| "clue_spy_votes_received": 5.375, | |
| "clue_suspicion_potential_psi": 4.666666666666667, | |
| "clue_total_valid_votes": 7.5, | |
| "completion_length": 267.59375, | |
| "epoch": 1.4285714285714286, | |
| "grad_norm": 1.3427386283874512, | |
| "kl": 0.0283355712890625, | |
| "learning_rate": 3.5714285714285718e-06, | |
| "loss": 0.002, | |
| "loss_interactive_phase": 0.0019903209613403305, | |
| "reward": 0.34910156111012686, | |
| "reward_original_clue_max": 0.21486971111570505, | |
| "reward_original_clue_mean": -4.395339522032804e-09, | |
| "reward_original_clue_min": -0.3329915751254489, | |
| "reward_original_clue_std": 0.2213489408298035, | |
| "reward_original_decision_max": 2.1412500000000003, | |
| "reward_original_decision_mean": 1.39640625, | |
| "reward_original_decision_min": -0.052499999999999936, | |
| "reward_original_decision_std": 0.777539948847112, | |
| "reward_original_overall_mean": 0.6982031228023303, | |
| "reward_original_overall_std": 1.1362766329525933, | |
| "reward_std": 0.8716088880016916, | |
| "step": 10 | |
| }, | |
| { | |
| "advantages_interactive_phase": 7.450580596923828e-09, | |
| "clip_ratio": 0.007512057200074196, | |
| "clue_civilian_adjusted_reward_mean": 0.029820853200277782, | |
| "clue_civilian_advantage_adjustment": -0.14240136902194445, | |
| "clue_civilian_baseline": 0.14240136902194445, | |
| "clue_civilian_raw_reward_mean": 0.17222222222222225, | |
| "clue_civilian_votes_avg": 0.45833333333333337, | |
| "clue_invalid_votes": 0.375, | |
| "clue_na_votes": 0.625, | |
| "clue_spy_adjusted_reward": -0.08946257515700651, | |
| "clue_spy_advantage_adjustment": 0.42720409150966016, | |
| "clue_spy_baseline": -0.42720409150966016, | |
| "clue_spy_raw_reward": -0.5166666666666667, | |
| "clue_spy_votes_received": 5.625, | |
| "clue_suspicion_potential_psi": 5.166666666666667, | |
| "clue_total_valid_votes": 7.0, | |
| "completion_length": 345.90625, | |
| "epoch": 1.5714285714285714, | |
| "grad_norm": 1.4723293781280518, | |
| "kl": 0.03619384765625, | |
| "learning_rate": 3.928571428571429e-06, | |
| "loss": 0.0017, | |
| "loss_interactive_phase": 0.0016628647717880085, | |
| "reward": 0.37558594025302955, | |
| "reward_original_clue_max": 0.16687582015889904, | |
| "reward_original_clue_mean": -3.8890432892298155e-09, | |
| "reward_original_clue_min": -0.2556842087822944, | |
| "reward_original_clue_std": 0.1723200812438524, | |
| "reward_original_decision_max": 2.49, | |
| "reward_original_decision_mean": 1.50234375, | |
| "reward_original_decision_min": -0.1274999999999999, | |
| "reward_original_decision_std": 0.9362325016572549, | |
| "reward_original_overall_mean": 0.7511718730554785, | |
| "reward_original_overall_std": 1.2049933860634257, | |
| "reward_std": 1.0033885483907903, | |
| "step": 11 | |
| }, | |
| { | |
| "advantages_interactive_phase": 4.249159246683121e-09, | |
| "clip_ratio": 0.00809995060262736, | |
| "clue_civilian_adjusted_reward_mean": 0.022188266754369242, | |
| "clue_civilian_advantage_adjustment": -0.187533955467853, | |
| "clue_civilian_baseline": 0.187533955467853, | |
| "clue_civilian_raw_reward_mean": 0.20972222222222225, | |
| "clue_civilian_votes_avg": 0.33333333333333337, | |
| "clue_invalid_votes": 0.125, | |
| "clue_na_votes": 0.25, | |
| "clue_spy_adjusted_reward": -0.0665648172282207, | |
| "clue_spy_advantage_adjustment": 0.5626018494384459, | |
| "clue_spy_baseline": -0.5626018494384459, | |
| "clue_spy_raw_reward": -0.6291666666666668, | |
| "clue_spy_votes_received": 6.625, | |
| "clue_suspicion_potential_psi": 6.291666666666667, | |
| "clue_total_valid_votes": 7.625, | |
| "completion_length": 252.8515625, | |
| "epoch": 1.7142857142857144, | |
| "grad_norm": 0.6337663531303406, | |
| "kl": 0.0308074951171875, | |
| "learning_rate": 4.2857142857142855e-06, | |
| "loss": 0.0016, | |
| "loss_interactive_phase": 0.0016085498355096206, | |
| "reward": 0.4791796885642602, | |
| "reward_original_clue_max": 0.13644402464982577, | |
| "reward_original_clue_mean": -4.2412782478781774e-09, | |
| "reward_original_clue_min": -0.2099872417903439, | |
| "reward_original_clue_std": 0.13869593786155765, | |
| "reward_original_decision_max": 2.49, | |
| "reward_original_decision_mean": 1.91671875, | |
| "reward_original_decision_min": 0.2962500000000001, | |
| "reward_original_decision_std": 0.7221503219446256, | |
| "reward_original_overall_mean": 0.9583593728793611, | |
| "reward_original_overall_std": 1.2329093618898852, | |
| "reward_std": 0.8671444384671382, | |
| "step": 12 | |
| }, | |
| { | |
| "advantages_interactive_phase": 1.862645149230957e-08, | |
| "clip_ratio": 0.0037154321908019483, | |
| "clue_civilian_adjusted_reward_mean": -0.0063334161387741394, | |
| "clue_civilian_advantage_adjustment": -0.17300008280544082, | |
| "clue_civilian_baseline": 0.1730000828054408, | |
| "clue_civilian_raw_reward_mean": 0.16666666666666663, | |
| "clue_civilian_votes_avg": 0.75, | |
| "clue_invalid_votes": 0.0, | |
| "clue_na_votes": 0.0, | |
| "clue_spy_adjusted_reward": 0.019000245031720445, | |
| "clue_spy_advantage_adjustment": 0.5190002450317205, | |
| "clue_spy_baseline": -0.5190002450317205, | |
| "clue_spy_raw_reward": -0.5, | |
| "clue_spy_votes_received": 5.75, | |
| "clue_suspicion_potential_psi": 5.000000000000001, | |
| "clue_total_valid_votes": 8.0, | |
| "completion_length": 337.15625, | |
| "epoch": 1.8571428571428572, | |
| "grad_norm": 0.6522341370582581, | |
| "kl": 0.060089111328125, | |
| "learning_rate": 4.642857142857144e-06, | |
| "loss": 0.0026, | |
| "loss_interactive_phase": 0.002614069191622548, | |
| "reward": 0.3841406341016882, | |
| "reward_original_clue_max": 0.2507983958494952, | |
| "reward_original_clue_mean": -8.461504942894327e-10, | |
| "reward_original_clue_min": -0.3617781823290228, | |
| "reward_original_clue_std": 0.23592987163167928, | |
| "reward_original_decision_max": 1.6425, | |
| "reward_original_decision_mean": 1.5365625, | |
| "reward_original_decision_min": 1.21875, | |
| "reward_original_decision_std": 0.183489132426828, | |
| "reward_original_overall_mean": 0.7682812495769249, | |
| "reward_original_overall_std": 1.0923944210890442, | |
| "reward_std": 0.6130125272216156, | |
| "step": 13 | |
| }, | |
| { | |
| "advantages_interactive_phase": 1.6996636986732483e-08, | |
| "clip_ratio": 0.00847951346077025, | |
| "clue_civilian_adjusted_reward_mean": -0.008037120541629665, | |
| "clue_civilian_advantage_adjustment": -0.1774815649860741, | |
| "clue_civilian_baseline": 0.1774815649860741, | |
| "clue_civilian_raw_reward_mean": 0.16944444444444443, | |
| "clue_civilian_votes_avg": 0.6666666666666667, | |
| "clue_invalid_votes": 0.0, | |
| "clue_na_votes": 0.25, | |
| "clue_spy_adjusted_reward": 0.024111363006878495, | |
| "clue_spy_advantage_adjustment": 0.5324446963402119, | |
| "clue_spy_baseline": -0.5324446963402119, | |
| "clue_spy_raw_reward": -0.5083333333333333, | |
| "clue_spy_votes_received": 5.75, | |
| "clue_suspicion_potential_psi": 5.083333333333334, | |
| "clue_total_valid_votes": 7.75, | |
| "completion_length": 263.8359375, | |
| "epoch": 2.0, | |
| "grad_norm": 0.6224793195724487, | |
| "kl": 0.05419921875, | |
| "learning_rate": 5e-06, | |
| "loss": 0.0029, | |
| "loss_interactive_phase": 0.0029011927836108953, | |
| "reward": 0.388828133584693, | |
| "reward_original_clue_max": 0.2085423028111688, | |
| "reward_original_clue_mean": 3.4549737831963967e-10, | |
| "reward_original_clue_min": -0.2378369221284988, | |
| "reward_original_clue_std": 0.17641129318052418, | |
| "reward_original_decision_max": 2.1412500000000003, | |
| "reward_original_decision_mean": 1.5553125000000003, | |
| "reward_original_decision_min": 0.3712500000000002, | |
| "reward_original_decision_std": 0.668291458945668, | |
| "reward_original_overall_mean": 0.777656250172749, | |
| "reward_original_overall_std": 1.157252940433482, | |
| "reward_std": 0.8331386712800759, | |
| "step": 14 | |
| }, | |
| { | |
| "advantages_interactive_phase": 1.6763806343078613e-08, | |
| "clip_ratio": 0.003465894202236086, | |
| "clue_civilian_adjusted_reward_mean": 0.023268842590345305, | |
| "clue_civilian_advantage_adjustment": -0.18367560185409915, | |
| "clue_civilian_baseline": 0.18367560185409915, | |
| "clue_civilian_raw_reward_mean": 0.20694444444444446, | |
| "clue_civilian_votes_avg": 0.41666666666666663, | |
| "clue_invalid_votes": 0.125, | |
| "clue_na_votes": 0.0, | |
| "clue_spy_adjusted_reward": -0.06980651791005336, | |
| "clue_spy_advantage_adjustment": 0.55102681542328, | |
| "clue_spy_baseline": -0.55102681542328, | |
| "clue_spy_raw_reward": -0.6208333333333333, | |
| "clue_spy_votes_received": 6.625, | |
| "clue_suspicion_potential_psi": 6.208333333333333, | |
| "clue_total_valid_votes": 7.875, | |
| "completion_length": 355.0625, | |
| "epoch": 2.142857142857143, | |
| "grad_norm": 0.7302604913711548, | |
| "kl": 0.08294677734375, | |
| "learning_rate": 5.357142857142857e-06, | |
| "loss": 0.0034, | |
| "loss_interactive_phase": 0.0033593956904951483, | |
| "reward": 0.4744921964982146, | |
| "reward_original_clue_max": 0.15940368854312534, | |
| "reward_original_clue_mean": 2.465245637836566e-09, | |
| "reward_original_clue_min": -0.24915193754399267, | |
| "reward_original_clue_std": 0.16536223128526092, | |
| "reward_original_decision_max": 2.06625, | |
| "reward_original_decision_mean": 1.8979687500000002, | |
| "reward_original_decision_min": 1.14375, | |
| "reward_original_decision_std": 0.34843519072601103, | |
| "reward_original_overall_mean": 0.9489843762326229, | |
| "reward_original_overall_std": 1.1879908640849528, | |
| "reward_std": 0.7276260579460828, | |
| "step": 15 | |
| }, | |
| { | |
| "advantages_interactive_phase": -5.820766091346741e-09, | |
| "clip_ratio": 0.006389049158315174, | |
| "clue_civilian_adjusted_reward_mean": -0.046890086079656224, | |
| "clue_civilian_advantage_adjustment": -0.1635567527463229, | |
| "clue_civilian_baseline": 0.1635567527463229, | |
| "clue_civilian_raw_reward_mean": 0.11666666666666668, | |
| "clue_civilian_votes_avg": 0.625, | |
| "clue_invalid_votes": 0.125, | |
| "clue_na_votes": 1.875, | |
| "clue_spy_adjusted_reward": 0.14067026040217548, | |
| "clue_spy_advantage_adjustment": 0.49067026040217543, | |
| "clue_spy_baseline": -0.49067026040217543, | |
| "clue_spy_raw_reward": -0.35000000000000003, | |
| "clue_spy_votes_received": 4.125, | |
| "clue_suspicion_potential_psi": 3.5, | |
| "clue_total_valid_votes": 6.0, | |
| "completion_length": 298.203125, | |
| "epoch": 2.2857142857142856, | |
| "grad_norm": 18.787372589111328, | |
| "kl": 0.0643310546875, | |
| "learning_rate": 5.7142857142857145e-06, | |
| "loss": 0.0227, | |
| "loss_interactive_phase": 0.02273198706097901, | |
| "reward": 0.2448046847248174, | |
| "reward_original_clue_max": 0.31508776732274035, | |
| "reward_original_clue_mean": 5.408017063682213e-10, | |
| "reward_original_clue_min": -0.296307593000221, | |
| "reward_original_clue_std": 0.24153634073282915, | |
| "reward_original_decision_max": 1.7925, | |
| "reward_original_decision_mean": 0.97921875, | |
| "reward_original_decision_min": 0.37125000000000014, | |
| "reward_original_decision_std": 0.4794102853203757, | |
| "reward_original_overall_mean": 0.4896093752704009, | |
| "reward_original_overall_std": 0.942394237518095, | |
| "reward_std": 0.732489835712951, | |
| "step": 16 | |
| }, | |
| { | |
| "advantages_interactive_phase": 3.91155481338501e-08, | |
| "clip_ratio": 0.003673899220302701, | |
| "clue_civilian_adjusted_reward_mean": 0.04772389224517483, | |
| "clue_civilian_advantage_adjustment": -0.16199832997704738, | |
| "clue_civilian_baseline": 0.16199832997704738, | |
| "clue_civilian_raw_reward_mean": 0.2097222222222222, | |
| "clue_civilian_votes_avg": 0.20833333333333331, | |
| "clue_invalid_votes": 0.375, | |
| "clue_na_votes": 0.5, | |
| "clue_spy_adjusted_reward": -0.1431716761674981, | |
| "clue_spy_advantage_adjustment": 0.4859949904991686, | |
| "clue_spy_baseline": -0.4859949904991686, | |
| "clue_spy_raw_reward": -0.6291666666666668, | |
| "clue_spy_votes_received": 6.5, | |
| "clue_suspicion_potential_psi": 6.291666666666667, | |
| "clue_total_valid_votes": 7.125, | |
| "completion_length": 363.796875, | |
| "epoch": 2.4285714285714284, | |
| "grad_norm": 1.1137752532958984, | |
| "kl": 0.099365234375, | |
| "learning_rate": 6.071428571428571e-06, | |
| "loss": 0.0042, | |
| "loss_interactive_phase": 0.0041947553982026875, | |
| "reward": 0.4659375195932758, | |
| "reward_original_clue_max": 0.13827404245616234, | |
| "reward_original_clue_mean": 1.420066000554221e-10, | |
| "reward_original_clue_min": -0.25038849304515226, | |
| "reward_original_clue_std": 0.16193395395069143, | |
| "reward_original_decision_max": 2.1412500000000003, | |
| "reward_original_decision_mean": 1.86375, | |
| "reward_original_decision_min": 0.7200000000000001, | |
| "reward_original_decision_std": 0.50161616427093, | |
| "reward_original_overall_mean": 0.9318750000710034, | |
| "reward_original_overall_std": 1.1870721598954623, | |
| "reward_std": 0.8607973168537828, | |
| "step": 17 | |
| }, | |
| { | |
| "advantages_interactive_phase": 2.837623469531536e-09, | |
| "clip_ratio": 0.00735139346215874, | |
| "clue_civilian_adjusted_reward_mean": -0.0017000992579862158, | |
| "clue_civilian_advantage_adjustment": -0.19475565481354176, | |
| "clue_civilian_baseline": 0.19475565481354176, | |
| "clue_civilian_raw_reward_mean": 0.19305555555555554, | |
| "clue_civilian_votes_avg": 0.20833333333333331, | |
| "clue_invalid_votes": 0.5, | |
| "clue_na_votes": 0.875, | |
| "clue_spy_adjusted_reward": 0.0051002900798029355, | |
| "clue_spy_advantage_adjustment": 0.5842669567464696, | |
| "clue_spy_baseline": -0.5842669567464696, | |
| "clue_spy_raw_reward": -0.5791666666666666, | |
| "clue_spy_votes_received": 6.0, | |
| "clue_suspicion_potential_psi": 5.791666666666667, | |
| "clue_total_valid_votes": 6.625, | |
| "completion_length": 296.6875, | |
| "epoch": 2.571428571428571, | |
| "grad_norm": 0.6697026491165161, | |
| "kl": 0.083984375, | |
| "learning_rate": 6.4285714285714295e-06, | |
| "loss": 0.0038, | |
| "loss_interactive_phase": 0.003771388641325757, | |
| "reward": 0.417656250937927, | |
| "reward_original_clue_max": 0.112484931462635, | |
| "reward_original_clue_mean": -1.923538928399332e-09, | |
| "reward_original_clue_min": -0.12320458815006033, | |
| "reward_original_clue_std": 0.09749921717912478, | |
| "reward_original_decision_max": 2.49, | |
| "reward_original_decision_mean": 1.6706250000000002, | |
| "reward_original_decision_min": -0.27749999999999997, | |
| "reward_original_decision_std": 1.025463516531797, | |
| "reward_original_overall_mean": 0.8353124990382306, | |
| "reward_original_overall_std": 1.2562141688103865, | |
| "reward_std": 0.9142232616213656, | |
| "step": 18 | |
| }, | |
| { | |
| "advantages_interactive_phase": 2.421438694000244e-08, | |
| "clip_ratio": 0.00690445041982457, | |
| "clue_civilian_adjusted_reward_mean": -0.02088481382776897, | |
| "clue_civilian_advantage_adjustment": -0.19588481382776896, | |
| "clue_civilian_baseline": 0.19588481382776896, | |
| "clue_civilian_raw_reward_mean": 0.17500000000000002, | |
| "clue_civilian_votes_avg": 0.375, | |
| "clue_invalid_votes": 0.125, | |
| "clue_na_votes": 1.125, | |
| "clue_spy_adjusted_reward": 0.06265442930658618, | |
| "clue_spy_advantage_adjustment": 0.5876544293065862, | |
| "clue_spy_baseline": -0.5876544293065862, | |
| "clue_spy_raw_reward": -0.525, | |
| "clue_spy_votes_received": 5.625, | |
| "clue_suspicion_potential_psi": 5.249999999999999, | |
| "clue_total_valid_votes": 6.75, | |
| "completion_length": 345.75, | |
| "epoch": 2.7142857142857144, | |
| "grad_norm": 1.3586255311965942, | |
| "kl": 0.135986328125, | |
| "learning_rate": 6.785714285714287e-06, | |
| "loss": 0.006, | |
| "loss_interactive_phase": 0.00598463078495115, | |
| "reward": 0.38964844884614847, | |
| "reward_original_clue_max": 0.19217094683455577, | |
| "reward_original_clue_mean": -3.044180179635758e-09, | |
| "reward_original_clue_min": -0.18790133135573855, | |
| "reward_original_clue_std": 0.15165388026680496, | |
| "reward_original_decision_max": 2.1412500000000003, | |
| "reward_original_decision_mean": 1.55859375, | |
| "reward_original_decision_min": 0.3712500000000002, | |
| "reward_original_decision_std": 0.6276860023276215, | |
| "reward_original_overall_mean": 0.77929687347791, | |
| "reward_original_overall_std": 1.1151671426286367, | |
| "reward_std": 0.8916601099963023, | |
| "step": 19 | |
| }, | |
| { | |
| "advantages_interactive_phase": 1.1408701539039612e-08, | |
| "clip_ratio": 0.0066332001879345626, | |
| "clue_civilian_adjusted_reward_mean": -0.014852289498390544, | |
| "clue_civilian_advantage_adjustment": -0.15790784505394606, | |
| "clue_civilian_baseline": 0.1579078450539461, | |
| "clue_civilian_raw_reward_mean": 0.14305555555555555, | |
| "clue_civilian_votes_avg": 0.7083333333333334, | |
| "clue_invalid_votes": 0.0, | |
| "clue_na_votes": 0.875, | |
| "clue_spy_adjusted_reward": 0.04455685833978199, | |
| "clue_spy_advantage_adjustment": 0.4737235250064487, | |
| "clue_spy_baseline": -0.47372352500644876, | |
| "clue_spy_raw_reward": -0.4291666666666667, | |
| "clue_spy_votes_received": 5.0, | |
| "clue_suspicion_potential_psi": 4.291666666666667, | |
| "clue_total_valid_votes": 7.125, | |
| "completion_length": 291.84375, | |
| "epoch": 2.857142857142857, | |
| "grad_norm": 0.7186980247497559, | |
| "kl": 0.120361328125, | |
| "learning_rate": 7.1428571428571436e-06, | |
| "loss": 0.0053, | |
| "loss_interactive_phase": 0.005272059410344809, | |
| "reward": 0.321093755069639, | |
| "reward_original_clue_max": 0.23781143176053088, | |
| "reward_original_clue_mean": -2.538847402276667e-09, | |
| "reward_original_clue_min": -0.3372735295858061, | |
| "reward_original_clue_std": 0.22314844267377926, | |
| "reward_original_decision_max": 2.1412500000000003, | |
| "reward_original_decision_mean": 1.2843750000000003, | |
| "reward_original_decision_min": 0.44625000000000015, | |
| "reward_original_decision_std": 0.682584747416193, | |
| "reward_original_overall_mean": 0.6421874987305765, | |
| "reward_original_overall_std": 1.0984288921667873, | |
| "reward_std": 0.8115738822191327, | |
| "step": 20 | |
| }, | |
| { | |
| "advantages_interactive_phase": 1.862645149230957e-09, | |
| "clip_ratio": 0.011862117098644376, | |
| "clue_civilian_adjusted_reward_mean": -0.011171407247594729, | |
| "clue_civilian_advantage_adjustment": -0.1500602961364836, | |
| "clue_civilian_baseline": 0.1500602961364836, | |
| "clue_civilian_raw_reward_mean": 0.1388888888888889, | |
| "clue_civilian_votes_avg": 0.4583333333333333, | |
| "clue_invalid_votes": 0.125, | |
| "clue_na_votes": 1.875, | |
| "clue_spy_adjusted_reward": 0.03351421315180729, | |
| "clue_spy_advantage_adjustment": 0.450180879818474, | |
| "clue_spy_baseline": -0.450180879818474, | |
| "clue_spy_raw_reward": -0.41666666666666674, | |
| "clue_spy_votes_received": 4.625, | |
| "clue_suspicion_potential_psi": 4.166666666666667, | |
| "clue_total_valid_votes": 6.0, | |
| "completion_length": 341.484375, | |
| "epoch": 3.0, | |
| "grad_norm": 2.092695713043213, | |
| "kl": 0.1983642578125, | |
| "learning_rate": 7.500000000000001e-06, | |
| "loss": 0.0115, | |
| "loss_interactive_phase": 0.011498338542878628, | |
| "reward": 0.29777343789438654, | |
| "reward_original_clue_max": 0.180558073555015, | |
| "reward_original_clue_mean": -2.147744222486675e-09, | |
| "reward_original_clue_min": -0.16238193431746906, | |
| "reward_original_clue_std": 0.138092222998997, | |
| "reward_original_decision_max": 1.7925, | |
| "reward_original_decision_mean": 1.19109375, | |
| "reward_original_decision_min": -0.8249999999999998, | |
| "reward_original_decision_std": 1.0596252090888885, | |
| "reward_original_overall_mean": 0.5955468739261279, | |
| "reward_original_overall_std": 1.0734861999424956, | |
| "reward_std": 1.0712655663327704, | |
| "step": 21 | |
| }, | |
| { | |
| "advantages_interactive_phase": 2.3166649043560028e-08, | |
| "clip_ratio": 0.007374117994913831, | |
| "clue_civilian_adjusted_reward_mean": -0.013323967362128385, | |
| "clue_civilian_advantage_adjustment": -0.1397128562510173, | |
| "clue_civilian_baseline": 0.1397128562510173, | |
| "clue_civilian_raw_reward_mean": 0.1263888888888889, | |
| "clue_civilian_votes_avg": 0.8333333333333333, | |
| "clue_invalid_votes": 0.25, | |
| "clue_na_votes": 0.625, | |
| "clue_spy_adjusted_reward": 0.03997188807820764, | |
| "clue_spy_advantage_adjustment": 0.4191385547448744, | |
| "clue_spy_baseline": -0.4191385547448744, | |
| "clue_spy_raw_reward": -0.3791666666666667, | |
| "clue_spy_votes_received": 4.625, | |
| "clue_suspicion_potential_psi": 3.7916666666666665, | |
| "clue_total_valid_votes": 7.125, | |
| "completion_length": 314.3828125, | |
| "epoch": 3.142857142857143, | |
| "grad_norm": 0.7247025966644287, | |
| "kl": 0.14990234375, | |
| "learning_rate": 7.857142857142858e-06, | |
| "loss": 0.0065, | |
| "loss_interactive_phase": 0.006495335896033794, | |
| "reward": 0.27199219820781345, | |
| "reward_original_clue_max": 0.20973680630488367, | |
| "reward_original_clue_mean": -3.5020443879887897e-09, | |
| "reward_original_clue_min": -0.25500620188084894, | |
| "reward_original_clue_std": 0.17895936766287218, | |
| "reward_original_decision_max": 2.06625, | |
| "reward_original_decision_mean": 1.0879687500000002, | |
| "reward_original_decision_min": -0.55125, | |
| "reward_original_decision_std": 1.0594716886601967, | |
| "reward_original_overall_mean": 0.5439843732489779, | |
| "reward_original_overall_std": 1.143192035744574, | |
| "reward_std": 0.8779618775670658, | |
| "step": 22 | |
| }, | |
| { | |
| "advantages_interactive_phase": 1.210719347000122e-08, | |
| "clip_ratio": 0.009816105710342526, | |
| "clue_civilian_adjusted_reward_mean": 0.012905466579270795, | |
| "clue_civilian_advantage_adjustment": -0.13431675564295142, | |
| "clue_civilian_baseline": 0.13431675564295142, | |
| "clue_civilian_raw_reward_mean": 0.14722222222222223, | |
| "clue_civilian_votes_avg": 0.8333333333333334, | |
| "clue_invalid_votes": 0.25, | |
| "clue_na_votes": 0.0, | |
| "clue_spy_adjusted_reward": -0.03871641069401783, | |
| "clue_spy_advantage_adjustment": 0.4029502559726489, | |
| "clue_spy_baseline": -0.4029502559726489, | |
| "clue_spy_raw_reward": -0.44166666666666665, | |
| "clue_spy_votes_received": 5.25, | |
| "clue_suspicion_potential_psi": 4.416666666666667, | |
| "clue_total_valid_votes": 7.75, | |
| "completion_length": 349.46875, | |
| "epoch": 3.2857142857142856, | |
| "grad_norm": 1.495042085647583, | |
| "kl": 0.20458984375, | |
| "learning_rate": 8.214285714285714e-06, | |
| "loss": 0.0098, | |
| "loss_interactive_phase": 0.009842151892371476, | |
| "reward": 0.3264843803688339, | |
| "reward_original_clue_max": 0.18625860761540888, | |
| "reward_original_clue_mean": -2.7390513624575563e-09, | |
| "reward_original_clue_min": -0.25603524557359103, | |
| "reward_original_clue_std": 0.16967942428353516, | |
| "reward_original_decision_max": 2.49, | |
| "reward_original_decision_mean": 1.3059375000000002, | |
| "reward_original_decision_min": -0.62625, | |
| "reward_original_decision_std": 1.2001143050462124, | |
| "reward_original_overall_mean": 0.6529687486304744, | |
| "reward_original_overall_std": 1.2300136329570874, | |
| "reward_std": 1.0827139959652952, | |
| "step": 23 | |
| }, | |
| { | |
| "advantages_interactive_phase": 2.0721927285194397e-08, | |
| "clip_ratio": 0.007756809587590396, | |
| "clue_civilian_adjusted_reward_mean": -0.04694887868266013, | |
| "clue_civilian_advantage_adjustment": -0.11500443423821569, | |
| "clue_civilian_baseline": 0.11500443423821569, | |
| "clue_civilian_raw_reward_mean": 0.06805555555555555, | |
| "clue_civilian_votes_avg": 1.3333333333333333, | |
| "clue_invalid_votes": 0.5, | |
| "clue_na_votes": 0.125, | |
| "clue_spy_adjusted_reward": 0.14084662848473525, | |
| "clue_spy_advantage_adjustment": 0.345013295151402, | |
| "clue_spy_baseline": -0.345013295151402, | |
| "clue_spy_raw_reward": -0.20416666666666672, | |
| "clue_spy_votes_received": 3.375, | |
| "clue_suspicion_potential_psi": 2.041666666666667, | |
| "clue_total_valid_votes": 7.375, | |
| "completion_length": 362.4375, | |
| "epoch": 3.4285714285714284, | |
| "grad_norm": 0.721821129322052, | |
| "kl": 0.15087890625, | |
| "learning_rate": 8.571428571428571e-06, | |
| "loss": 0.0064, | |
| "loss_interactive_phase": 0.006402550439815968, | |
| "reward": 0.12550782238826086, | |
| "reward_original_clue_max": 0.2615399846222246, | |
| "reward_original_clue_mean": -1.8908112909589436e-09, | |
| "reward_original_clue_min": -0.29680578730801793, | |
| "reward_original_clue_std": 0.21245343365808805, | |
| "reward_original_decision_max": 2.06625, | |
| "reward_original_decision_mean": 0.50203125, | |
| "reward_original_decision_min": -1.125, | |
| "reward_original_decision_std": 1.2899370730874167, | |
| "reward_original_overall_mean": 0.25101562405459443, | |
| "reward_original_overall_std": 1.1078002049178977, | |
| "reward_std": 0.8967364060953804, | |
| "step": 24 | |
| }, | |
| { | |
| "advantages_interactive_phase": 1.1175870895385742e-08, | |
| "clip_ratio": 0.0030952056986279786, | |
| "clue_civilian_adjusted_reward_mean": 0.05641245992546336, | |
| "clue_civilian_advantage_adjustment": -0.1352542067412033, | |
| "clue_civilian_baseline": 0.1352542067412033, | |
| "clue_civilian_raw_reward_mean": 0.19166666666666668, | |
| "clue_civilian_votes_avg": 0.5, | |
| "clue_invalid_votes": 0.25, | |
| "clue_na_votes": 0.0, | |
| "clue_spy_adjusted_reward": -0.16923738346885245, | |
| "clue_spy_advantage_adjustment": 0.40576261653114754, | |
| "clue_spy_baseline": -0.40576261653114754, | |
| "clue_spy_raw_reward": -0.575, | |
| "clue_spy_votes_received": 6.25, | |
| "clue_suspicion_potential_psi": 5.75, | |
| "clue_total_valid_votes": 7.75, | |
| "completion_length": 378.28125, | |
| "epoch": 3.571428571428571, | |
| "grad_norm": 1.2147719860076904, | |
| "kl": 0.2017822265625, | |
| "learning_rate": 8.92857142857143e-06, | |
| "loss": 0.0086, | |
| "loss_interactive_phase": 0.008552196552045643, | |
| "reward": 0.4324218803571566, | |
| "reward_original_clue_max": 0.17644798867648273, | |
| "reward_original_clue_mean": -9.231155991516748e-10, | |
| "reward_original_clue_min": -0.3267729122198718, | |
| "reward_original_clue_std": 0.20414503627659822, | |
| "reward_original_decision_max": 2.49, | |
| "reward_original_decision_mean": 1.7296875000000003, | |
| "reward_original_decision_min": 0.645, | |
| "reward_original_decision_std": 0.6397808895425074, | |
| "reward_original_overall_mean": 0.8648437495384423, | |
| "reward_original_overall_std": 1.219408722443206, | |
| "reward_std": 0.876965590677074, | |
| "step": 25 | |
| }, | |
| { | |
| "advantages_interactive_phase": 2.421438694000244e-08, | |
| "clip_ratio": 0.005824881722219288, | |
| "clue_civilian_adjusted_reward_mean": -0.007093781877094507, | |
| "clue_civilian_advantage_adjustment": -0.13209378187709453, | |
| "clue_civilian_baseline": 0.13209378187709453, | |
| "clue_civilian_raw_reward_mean": 0.125, | |
| "clue_civilian_votes_avg": 1.0, | |
| "clue_invalid_votes": 0.25, | |
| "clue_na_votes": 0.0, | |
| "clue_spy_adjusted_reward": 0.021281343083350023, | |
| "clue_spy_advantage_adjustment": 0.39628134308335006, | |
| "clue_spy_baseline": -0.39628134308335006, | |
| "clue_spy_raw_reward": -0.375, | |
| "clue_spy_votes_received": 4.75, | |
| "clue_suspicion_potential_psi": 3.7499999999999996, | |
| "clue_total_valid_votes": 7.75, | |
| "completion_length": 333.84375, | |
| "epoch": 3.7142857142857144, | |
| "grad_norm": 0.8488478660583496, | |
| "kl": 0.18359375, | |
| "learning_rate": 9.285714285714288e-06, | |
| "loss": 0.0078, | |
| "loss_interactive_phase": 0.007767571427393705, | |
| "reward": 0.27351563694794767, | |
| "reward_original_clue_max": 0.25575890463160234, | |
| "reward_original_clue_mean": -6.369833735996033e-10, | |
| "reward_original_clue_min": -0.3290713434253468, | |
| "reward_original_clue_std": 0.22778135711245912, | |
| "reward_original_decision_max": 2.06625, | |
| "reward_original_decision_mean": 1.0940625000000002, | |
| "reward_original_decision_min": -0.1274999999999999, | |
| "reward_original_decision_std": 0.8632276294383053, | |
| "reward_original_overall_mean": 0.5470312496815084, | |
| "reward_original_overall_std": 1.1264277840598063, | |
| "reward_std": 0.8660217167160729, | |
| "step": 26 | |
| }, | |
| { | |
| "advantages_interactive_phase": -7.450580596923828e-09, | |
| "clip_ratio": 0.0031035091960802674, | |
| "clue_civilian_adjusted_reward_mean": 0.03245208673288981, | |
| "clue_civilian_advantage_adjustment": -0.1689368021559991, | |
| "clue_civilian_baseline": 0.1689368021559991, | |
| "clue_civilian_raw_reward_mean": 0.2013888888888889, | |
| "clue_civilian_votes_avg": 0.4583333333333333, | |
| "clue_invalid_votes": 0.125, | |
| "clue_na_votes": 0.0, | |
| "clue_spy_adjusted_reward": -0.09735625958223877, | |
| "clue_spy_advantage_adjustment": 0.5068104070844279, | |
| "clue_spy_baseline": -0.5068104070844279, | |
| "clue_spy_raw_reward": -0.6041666666666666, | |
| "clue_spy_votes_received": 6.5, | |
| "clue_suspicion_potential_psi": 6.041666666666667, | |
| "clue_total_valid_votes": 7.875, | |
| "completion_length": 306.40625, | |
| "epoch": 3.857142857142857, | |
| "grad_norm": 1.9208674430847168, | |
| "kl": 0.3026123046875, | |
| "learning_rate": 9.642857142857144e-06, | |
| "loss": 0.0125, | |
| "loss_interactive_phase": 0.01246593298856169, | |
| "reward": 0.46124999631323677, | |
| "reward_original_clue_max": 0.1545202636139022, | |
| "reward_original_clue_mean": 1.5410765849196983e-10, | |
| "reward_original_clue_min": -0.24859110312991783, | |
| "reward_original_clue_std": 0.16558966332718913, | |
| "reward_original_decision_max": 2.49, | |
| "reward_original_decision_mean": 1.845, | |
| "reward_original_decision_min": 0.7200000000000002, | |
| "reward_original_decision_std": 0.7004523304839636, | |
| "reward_original_overall_mean": 0.9225000000770541, | |
| "reward_original_overall_std": 1.2500439582236307, | |
| "reward_std": 0.8922832234684476, | |
| "step": 27 | |
| }, | |
| { | |
| "advantages_interactive_phase": 9.66247171163559e-09, | |
| "clip_ratio": 0.004918134000035934, | |
| "clue_civilian_adjusted_reward_mean": -0.03922444833702034, | |
| "clue_civilian_advantage_adjustment": -0.168391115003687, | |
| "clue_civilian_baseline": 0.168391115003687, | |
| "clue_civilian_raw_reward_mean": 0.12916666666666665, | |
| "clue_civilian_votes_avg": 1.0, | |
| "clue_invalid_votes": 0.125, | |
| "clue_na_votes": 0.0, | |
| "clue_spy_adjusted_reward": 0.11767334568750121, | |
| "clue_spy_advantage_adjustment": 0.5051733456875013, | |
| "clue_spy_baseline": -0.5051733456875013, | |
| "clue_spy_raw_reward": -0.3875000000000001, | |
| "clue_spy_votes_received": 4.875, | |
| "clue_suspicion_potential_psi": 3.8750000000000004, | |
| "clue_total_valid_votes": 7.875, | |
| "completion_length": 290.8359375, | |
| "epoch": 4.0, | |
| "grad_norm": 0.778830885887146, | |
| "kl": 0.2080078125, | |
| "learning_rate": 1e-05, | |
| "loss": 0.0085, | |
| "loss_interactive_phase": 0.008535244385711849, | |
| "reward": 0.29144531737351337, | |
| "reward_original_clue_max": 0.2624036269472061, | |
| "reward_original_clue_mean": 1.691100571209192e-10, | |
| "reward_original_clue_min": -0.2589547295967252, | |
| "reward_original_clue_std": 0.20231139453953567, | |
| "reward_original_decision_max": 2.49, | |
| "reward_original_decision_mean": 1.16578125, | |
| "reward_original_decision_min": -0.05249999999999988, | |
| "reward_original_decision_std": 0.9492047780780623, | |
| "reward_original_overall_mean": 0.582890625084555, | |
| "reward_original_overall_std": 1.1522968480359421, | |
| "reward_std": 0.8734021281821482, | |
| "step": 28 | |
| }, | |
| { | |
| "advantages_interactive_phase": 2.60770320892334e-08, | |
| "clip_ratio": 0.004597238206770271, | |
| "clue_civilian_adjusted_reward_mean": -0.017886474267241367, | |
| "clue_civilian_advantage_adjustment": -0.13455314093390802, | |
| "clue_civilian_baseline": 0.13455314093390805, | |
| "clue_civilian_raw_reward_mean": 0.11666666666666667, | |
| "clue_civilian_votes_avg": 1.125, | |
| "clue_invalid_votes": 0.0, | |
| "clue_na_votes": 0.0, | |
| "clue_spy_adjusted_reward": 0.05365942077661086, | |
| "clue_spy_advantage_adjustment": 0.4036594207766109, | |
| "clue_spy_baseline": -0.4036594207766109, | |
| "clue_spy_raw_reward": -0.35, | |
| "clue_spy_votes_received": 4.625, | |
| "clue_suspicion_potential_psi": 3.5, | |
| "clue_total_valid_votes": 8.0, | |
| "completion_length": 304.515625, | |
| "epoch": 4.142857142857143, | |
| "grad_norm": 1.3750234842300415, | |
| "kl": 0.3173828125, | |
| "learning_rate": 9.999611462404874e-06, | |
| "loss": 0.0125, | |
| "loss_interactive_phase": 0.012478827498853207, | |
| "reward": 0.26496095041194656, | |
| "reward_original_clue_max": 0.26774659185022603, | |
| "reward_original_clue_mean": -5.062783100737156e-10, | |
| "reward_original_clue_min": -0.31947364534085654, | |
| "reward_original_clue_std": 0.23106213560541738, | |
| "reward_original_decision_max": 2.06625, | |
| "reward_original_decision_mean": 1.05984375, | |
| "reward_original_decision_min": -0.05249999999999988, | |
| "reward_original_decision_std": 0.8307205925127058, | |
| "reward_original_overall_mean": 0.529921874746861, | |
| "reward_original_overall_std": 1.1095292288061827, | |
| "reward_std": 0.8888411754368172, | |
| "step": 29 | |
| }, | |
| { | |
| "advantages_interactive_phase": 3.259629011154175e-09, | |
| "clip_ratio": 0.004411640649777837, | |
| "clue_civilian_adjusted_reward_mean": -0.019376439787606455, | |
| "clue_civilian_advantage_adjustment": -0.10826532867649533, | |
| "clue_civilian_baseline": 0.10826532867649533, | |
| "clue_civilian_raw_reward_mean": 0.08888888888888888, | |
| "clue_civilian_votes_avg": 1.3333333333333333, | |
| "clue_invalid_votes": 0.0, | |
| "clue_na_votes": 0.0, | |
| "clue_spy_adjusted_reward": 0.058129320628341516, | |
| "clue_spy_advantage_adjustment": 0.3247959872950082, | |
| "clue_spy_baseline": -0.3247959872950082, | |
| "clue_spy_raw_reward": -0.2666666666666667, | |
| "clue_spy_votes_received": 4.0, | |
| "clue_suspicion_potential_psi": 2.6666666666666665, | |
| "clue_total_valid_votes": 8.0, | |
| "completion_length": 261.4453125, | |
| "epoch": 4.285714285714286, | |
| "grad_norm": 0.8253775238990784, | |
| "kl": 0.2550048828125, | |
| "learning_rate": 9.998445910004082e-06, | |
| "loss": 0.0103, | |
| "loss_interactive_phase": 0.010343844187445939, | |
| "reward": 0.19875000170890966, | |
| "reward_original_clue_max": 0.2470865782057527, | |
| "reward_original_clue_mean": 3.1638053678631484e-10, | |
| "reward_original_clue_min": -0.258379067343329, | |
| "reward_original_clue_std": 0.19545089761013185, | |
| "reward_original_decision_max": 2.06625, | |
| "reward_original_decision_mean": 0.7949999999999999, | |
| "reward_original_decision_min": -0.47625000000000006, | |
| "reward_original_decision_std": 1.1225616644698808, | |
| "reward_original_overall_mean": 0.3975000001581903, | |
| "reward_original_overall_std": 1.1179099397348429, | |
| "reward_std": 0.8648673956089749, | |
| "step": 30 | |
| } | |
| ], | |
| "logging_steps": 1.0, | |
| "max_steps": 280, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 40, | |
| "save_steps": 5, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": false | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 1, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |