jarrelscy's picture
Publish GLM-5.3 NVFP4 / ARVQ v2 same_input fp16-block hybrid: non-cold structure seed (part 2)
868041e verified
Raw History Blame
22.8 kB
{
"layer": 29,
"target": "reference",
"recipe": "full_corpus_sequence_pass_v1",
"batch_tokens": 262144,
"sequence_order_sha256": "edebb22a5f7b34a6cbfa59c58aab9f111651d6ce0926212b5237dea3ef0ba485",
"training_sequence_count": 17580,
"training_passes": 0.8737237281109949,
"training_tokens_seen": 15728640,
"adaptive_schedule": {
"best": 0.08264907944194753,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 40,
"early_drop": false
},
"refinement": null,
"lr_decay_after": 45,
"lr_decay_factor": 0.25,
"export_only": false,
"training_output_export": "fixed validation/audit outputs only; full training trajectory export not performed",
"codebook_lr": 0.048,
"scale_lr": 0.032,
"quantized_movement": {
"codebook_changed": 30076,
"codebook_values": 1302528,
"scales_changed": 34279335,
"scale_values": 46891008
},
"initial_validation": {
"target_rel": 0.08589460018658307,
"reference_rel": 0.08589460018658307,
"windows": [
{
"first_row": 0,
"rows": 1024,
"reference_sse": 1636.73828125,
"reference_energy": 1292898.625
},
{
"first_row": 1024,
"rows": 1024,
"reference_sse": 3902.38720703125,
"reference_energy": 1265431.25
},
{
"first_row": 4096,
"rows": 1024,
"reference_sse": 62039.94140625,
"reference_energy": 2588239.5
},
{
"first_row": 5120,
"rows": 1024,
"reference_sse": 1913.6435546875,
"reference_energy": 1255318.0
},
{
"first_row": 8192,
"rows": 1024,
"reference_sse": 40413.34375,
"reference_energy": 1225761.375
},
{
"first_row": 9216,
"rows": 1024,
"reference_sse": 3418.66259765625,
"reference_energy": 3903470.5
},
{
"first_row": 12288,
"rows": 1024,
"reference_sse": 1703.2080078125,
"reference_energy": 1217064.5
},
{
"first_row": 13312,
"rows": 1024,
"reference_sse": 1396.5067138671875,
"reference_energy": 1307201.0
},
{
"first_row": 16384,
"rows": 1024,
"reference_sse": 1587.21533203125,
"reference_energy": 1285112.25
},
{
"first_row": 17408,
"rows": 1024,
"reference_sse": 2112.301513671875,
"reference_energy": 1314795.25
},
{
"first_row": 20480,
"rows": 1024,
"reference_sse": 1566.784423828125,
"reference_energy": 1313920.75
},
{
"first_row": 21504,
"rows": 1024,
"reference_sse": 5836.373046875,
"reference_energy": 2585153.5
},
{
"first_row": 24576,
"rows": 1024,
"reference_sse": 1837.7119140625,
"reference_energy": 1312118.125
},
{
"first_row": 25600,
"rows": 1024,
"reference_sse": 1627.858642578125,
"reference_energy": 1302486.0
},
{
"first_row": 28672,
"rows": 1024,
"reference_sse": 2288.64501953125,
"reference_energy": 1221143.25
},
{
"first_row": 29696,
"rows": 1024,
"reference_sse": 56318.515625,
"reference_energy": 1308295.0
}
]
},
"final_validation": {
"target_rel": 0.08264907944194753,
"reference_rel": 0.08264907944194753,
"windows": [
{
"first_row": 0,
"rows": 1024,
"reference_sse": 1638.7255859375,
"reference_energy": 1292898.625
},
{
"first_row": 1024,
"rows": 1024,
"reference_sse": 4015.84228515625,
"reference_energy": 1265431.25
},
{
"first_row": 4096,
"rows": 1024,
"reference_sse": 62685.484375,
"reference_energy": 2588239.5
},
{
"first_row": 5120,
"rows": 1024,
"reference_sse": 1906.4478759765625,
"reference_energy": 1255318.0
},
{
"first_row": 8192,
"rows": 1024,
"reference_sse": 32262.6953125,
"reference_energy": 1225761.375
},
{
"first_row": 9216,
"rows": 1024,
"reference_sse": 3339.60009765625,
"reference_energy": 3903470.5
},
{
"first_row": 12288,
"rows": 1024,
"reference_sse": 1937.5098876953125,
"reference_energy": 1217064.5
},
{
"first_row": 13312,
"rows": 1024,
"reference_sse": 1583.0517578125,
"reference_energy": 1307201.0
},
{
"first_row": 16384,
"rows": 1024,
"reference_sse": 1788.3348388671875,
"reference_energy": 1285112.25
},
{
"first_row": 17408,
"rows": 1024,
"reference_sse": 2348.896484375,
"reference_energy": 1314795.25
},
{
"first_row": 20480,
"rows": 1024,
"reference_sse": 1525.34228515625,
"reference_energy": 1313920.75
},
{
"first_row": 21504,
"rows": 1024,
"reference_sse": 4034.70654296875,
"reference_energy": 2585153.5
},
{
"first_row": 24576,
"rows": 1024,
"reference_sse": 1775.829345703125,
"reference_energy": 1312118.125
},
{
"first_row": 25600,
"rows": 1024,
"reference_sse": 1697.8494873046875,
"reference_energy": 1302486.0
},
{
"first_row": 28672,
"rows": 1024,
"reference_sse": 2290.2451171875,
"reference_energy": 1221143.25
},
{
"first_row": 29696,
"rows": 1024,
"reference_sse": 50711.9375,
"reference_energy": 1308295.0
}
]
},
"initial_reasoning_validation": null,
"final_reasoning_validation": null,
"validation_distribution": "reasoning only",
"serialized_validation": {
"target_rel": 0.08264907944194753,
"reference_rel": 0.08264907944194753
},
"development_audit": {
"target_rel": 0.05160641915131565,
"reference_rel": 0.05160641915131565,
"windows": [
{
"first_row": 2048,
"rows": 1024,
"reference_sse": 1901.242919921875,
"reference_energy": 1253199.5
},
{
"first_row": 3072,
"rows": 1024,
"reference_sse": 3130.791015625,
"reference_energy": 1324532.25
},
{
"first_row": 6144,
"rows": 1024,
"reference_sse": 1703.8809814453125,
"reference_energy": 1281360.25
},
{
"first_row": 7168,
"rows": 1024,
"reference_sse": 2668.68505859375,
"reference_energy": 3919132.75
},
{
"first_row": 10240,
"rows": 1024,
"reference_sse": 2687.737060546875,
"reference_energy": 2587723.75
},
{
"first_row": 11264,
"rows": 1024,
"reference_sse": 2187.04345703125,
"reference_energy": 1304963.0
},
{
"first_row": 14336,
"rows": 1024,
"reference_sse": 9483.9287109375,
"reference_energy": 2612513.75
},
{
"first_row": 15360,
"rows": 1024,
"reference_sse": 1605.490478515625,
"reference_energy": 1325034.0
},
{
"first_row": 18432,
"rows": 1024,
"reference_sse": 1750.0006103515625,
"reference_energy": 1220127.375
},
{
"first_row": 19456,
"rows": 1024,
"reference_sse": 1923.0625,
"reference_energy": 1319860.0
},
{
"first_row": 22528,
"rows": 1024,
"reference_sse": 1836.2498779296875,
"reference_energy": 1258317.375
},
{
"first_row": 23552,
"rows": 1024,
"reference_sse": 29135.189453125,
"reference_energy": 1277102.0
},
{
"first_row": 26624,
"rows": 1024,
"reference_sse": 1465.906982421875,
"reference_energy": 1254452.875
},
{
"first_row": 27648,
"rows": 1024,
"reference_sse": 2392.999755859375,
"reference_energy": 2606037.75
},
{
"first_row": 30720,
"rows": 1024,
"reference_sse": 1598.98876953125,
"reference_energy": 1226879.25
},
{
"first_row": 31744,
"rows": 1024,
"reference_sse": 6612.31787109375,
"reference_energy": 1295040.25
}
]
},
"initial_development_audit": {
"target_rel": 0.055049544452026075,
"reference_rel": 0.055049544452026075,
"windows": [
{
"first_row": 2048,
"rows": 1024,
"reference_sse": 1896.6710205078125,
"reference_energy": 1253199.5
},
{
"first_row": 3072,
"rows": 1024,
"reference_sse": 1937.46533203125,
"reference_energy": 1324532.25
},
{
"first_row": 6144,
"rows": 1024,
"reference_sse": 1706.2733154296875,
"reference_energy": 1281360.25
},
{
"first_row": 7168,
"rows": 1024,
"reference_sse": 2387.859375,
"reference_energy": 3919132.75
},
{
"first_row": 10240,
"rows": 1024,
"reference_sse": 2672.45556640625,
"reference_energy": 2587723.75
},
{
"first_row": 11264,
"rows": 1024,
"reference_sse": 2071.78515625,
"reference_energy": 1304963.0
},
{
"first_row": 14336,
"rows": 1024,
"reference_sse": 10070.2177734375,
"reference_energy": 2612513.75
},
{
"first_row": 15360,
"rows": 1024,
"reference_sse": 1403.4013671875,
"reference_energy": 1325034.0
},
{
"first_row": 18432,
"rows": 1024,
"reference_sse": 1745.14013671875,
"reference_energy": 1220127.375
},
{
"first_row": 19456,
"rows": 1024,
"reference_sse": 1924.4547119140625,
"reference_energy": 1319860.0
},
{
"first_row": 22528,
"rows": 1024,
"reference_sse": 2384.941650390625,
"reference_energy": 1258317.375
},
{
"first_row": 23552,
"rows": 1024,
"reference_sse": 37791.390625,
"reference_energy": 1277102.0
},
{
"first_row": 26624,
"rows": 1024,
"reference_sse": 1436.80322265625,
"reference_energy": 1254452.875
},
{
"first_row": 27648,
"rows": 1024,
"reference_sse": 2203.76708984375,
"reference_energy": 2606037.75
},
{
"first_row": 30720,
"rows": 1024,
"reference_sse": 1583.55859375,
"reference_energy": 1226879.25
},
{
"first_row": 31744,
"rows": 1024,
"reference_sse": 8806.875,
"reference_energy": 1295040.25
}
]
},
"best_step": 40,
"steps_run": 60,
"history": [
{
"step": 5,
"validation": {
"target_rel": 0.08571035207607355,
"reference_rel": 0.08571035207607355
},
"reasoning_validation": null,
"training_probe_rel": 0.880600886329777,
"batch_loss": 0.8132505267858505,
"best_step": 5,
"overfit_checks": 0,
"seconds": 47.698899269104004,
"adaptive_schedule": {
"best": 0.08571035207607355,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 5,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 0,
"codebook_values": 1302528,
"scales_changed": 14081705,
"scale_values": 46891008
}
},
{
"step": 10,
"validation": {
"target_rel": 0.08499342271946041,
"reference_rel": 0.08499342271946041
},
"reasoning_validation": null,
"training_probe_rel": 0.8799835479907436,
"batch_loss": 0.8050706535577774,
"best_step": 10,
"overfit_checks": 0,
"seconds": 96.42711997032166,
"adaptive_schedule": {
"best": 0.08499342271946041,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 10,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 34326,
"codebook_values": 1302528,
"scales_changed": 23095741,
"scale_values": 46891008
}
},
{
"step": 15,
"validation": {
"target_rel": 0.0838350495723564,
"reference_rel": 0.0838350495723564
},
"reasoning_validation": null,
"training_probe_rel": 0.8804745463589595,
"batch_loss": 0.950917199254036,
"best_step": 15,
"overfit_checks": 0,
"seconds": 153.21988368034363,
"adaptive_schedule": {
"best": 0.0838350495723564,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 15,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 27222,
"codebook_values": 1302528,
"scales_changed": 27475660,
"scale_values": 46891008
}
},
{
"step": 20,
"validation": {
"target_rel": 0.0831414496824238,
"reference_rel": 0.0831414496824238
},
"reasoning_validation": null,
"training_probe_rel": 0.8845416163407592,
"batch_loss": 0.9164185672998428,
"best_step": 20,
"overfit_checks": 0,
"seconds": 211.4613173007965,
"adaptive_schedule": {
"best": 0.0831414496824238,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 20,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 31867,
"codebook_values": 1302528,
"scales_changed": 30080873,
"scale_values": 46891008
}
},
{
"step": 25,
"validation": {
"target_rel": 0.08294073075359856,
"reference_rel": 0.08294073075359856
},
"reasoning_validation": null,
"training_probe_rel": 0.8817316132342565,
"batch_loss": 0.8480782508850098,
"best_step": 25,
"overfit_checks": 0,
"seconds": 262.4365997314453,
"adaptive_schedule": {
"best": 0.08294073075359856,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 25,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 32099,
"codebook_values": 1302528,
"scales_changed": 31724028,
"scale_values": 46891008
}
},
{
"step": 30,
"validation": {
"target_rel": 0.08345744938701834,
"reference_rel": 0.08345744938701834
},
"reasoning_validation": null,
"training_probe_rel": 0.8769087221884635,
"batch_loss": 0.7911168187856674,
"best_step": 25,
"overfit_checks": 1,
"seconds": 308.217413187027,
"adaptive_schedule": {
"best": 0.08294073075359856,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 1,
"last_best": 25,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 29598,
"codebook_values": 1302528,
"scales_changed": 32832462,
"scale_values": 46891008
}
},
{
"step": 35,
"validation": {
"target_rel": 0.08317251417552785,
"reference_rel": 0.08317251417552785
},
"reasoning_validation": null,
"training_probe_rel": 0.8771065440323194,
"batch_loss": 0.9147570133209229,
"best_step": 25,
"overfit_checks": 0,
"seconds": 357.731059551239,
"adaptive_schedule": {
"best": 0.08294073075359856,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 2,
"last_best": 25,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 28815,
"codebook_values": 1302528,
"scales_changed": 33645158,
"scale_values": 46891008
}
},
{
"step": 40,
"validation": {
"target_rel": 0.08264907944194753,
"reference_rel": 0.08264907944194753
},
"reasoning_validation": null,
"training_probe_rel": 0.8803671060082107,
"batch_loss": 0.7207415103912354,
"best_step": 40,
"overfit_checks": 0,
"seconds": 416.21785044670105,
"adaptive_schedule": {
"best": 0.08264907944194753,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 40,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 30076,
"codebook_values": 1302528,
"scales_changed": 34279335,
"scale_values": 46891008
}
},
{
"step": 45,
"validation": {
"target_rel": 0.08270545337696983,
"reference_rel": 0.08270545337696983
},
"reasoning_validation": null,
"training_probe_rel": 0.8771767613907334,
"batch_loss": 0.7422830611467361,
"best_step": 40,
"overfit_checks": 0,
"seconds": 463.2665596008301,
"adaptive_schedule": {
"best": 0.08264907944194753,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 40,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 29366,
"codebook_values": 1302528,
"scales_changed": 34775038,
"scale_values": 46891008
}
},
{
"step": 50,
"validation": {
"target_rel": 0.08267078498766953,
"reference_rel": 0.08267078498766953
},
"reasoning_validation": null,
"training_probe_rel": 0.8763662197526706,
"batch_loss": 0.8666548132896423,
"best_step": 40,
"overfit_checks": 0,
"seconds": 513.5701775550842,
"adaptive_schedule": {
"best": 0.08264907944194753,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 40,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 26835,
"codebook_values": 1302528,
"scales_changed": 34872602,
"scale_values": 46891008
}
},
{
"step": 55,
"validation": {
"target_rel": 0.08274457976209787,
"reference_rel": 0.08274457976209787
},
"reasoning_validation": null,
"training_probe_rel": 0.8758075715801756,
"batch_loss": 0.9579499959945679,
"best_step": 40,
"overfit_checks": 0,
"seconds": 559.786609172821,
"adaptive_schedule": {
"best": 0.08264907944194753,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 40,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 27532,
"codebook_values": 1302528,
"scales_changed": 34949680,
"scale_values": 46891008
}
},
{
"step": 60,
"validation": {
"target_rel": 0.08312712580315856,
"reference_rel": 0.08312712580315856
},
"reasoning_validation": null,
"training_probe_rel": 0.8743790430931386,
"batch_loss": 0.8302742540836334,
"best_step": 40,
"overfit_checks": 1,
"seconds": 622.0802729129791,
"adaptive_schedule": {
"best": 0.08264907944194753,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 40,
"early_drop": false
},
"adaptive_stop": true,
"quantized_movement": {
"codebook_changed": 27183,
"codebook_values": 1302528,
"scales_changed": 35015635,
"scale_values": 46891008
}
}
],
"index_reassignments": [
{
"method": "expert_parallel_output_gradient_prefix_backtracking_v1",
"proposals": 316,
"accepted": 281,
"changed_groups": 139695,
"training_before": 87372.625,
"training_after": 85451.0703125,
"acceptance": "proposal and separate training check batches",
"step": 20
},
{
"method": "expert_parallel_output_gradient_prefix_backtracking_v1",
"proposals": 316,
"accepted": 302,
"changed_groups": 150641,
"training_before": 44178.28125,
"training_after": 43727.0546875,
"acceptance": "proposal and separate training check batches",
"step": 40
},
{
"method": "expert_parallel_output_gradient_prefix_backtracking_v1",
"proposals": 313,
"accepted": 253,
"changed_groups": 113378,
"training_before": 61001.0859375,
"training_after": 60863.09375,
"acceptance": "proposal and separate training check batches",
"step": 60
}
],
"coverage_threshold": 256,
"checkpoint_selection": "held-out original reference trajectory for both arms",
"world_size": 8,
"microbatch": 65536,
"proposal_communication": "sparse routed rows",
"parallelism": "expert-sharded coupled output",
"arithmetic": "fp4_planes4_fp16_boundaries_v1",
"scope": "text-only, cached prompt partitions; historical audit is development only",
"train_rows": 18001846,
"validation_rows": 16384,
"audit_rows": 16384,
"complete": true
}