jarrelscy's picture
Publish GLM-5.3 NVFP4 / ARVQ v2 same_input fp16-block hybrid: non-cold structure seed (part 2)
868041e verified
Raw History Blame
22.6 kB
{
"layer": 41,
"target": "reference",
"recipe": "full_corpus_sequence_pass_v1",
"block_scale_dtype": "fp8_e4m3",
"batch_tokens": 262144,
"sequence_order_sha256": "edebb22a5f7b34a6cbfa59c58aab9f111651d6ce0926212b5237dea3ef0ba485",
"training_sequence_count": 17580,
"training_passes": 0.8737237281109949,
"training_tokens_seen": 15728640,
"adaptive_schedule": {
"best": 0.20210293694613474,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 35,
"early_drop": false
},
"refinement": null,
"lr_decay_after": 45,
"lr_decay_factor": 0.25,
"export_only": false,
"training_output_export": "fixed validation/audit outputs only; full training trajectory export not performed",
"codebook_lr": 0.048,
"scale_lr": 0.032,
"quantized_movement": {
"codebook_changed": 2336,
"codebook_values": 1425408,
"scales_changed": 32618897,
"scale_values": 51314688
},
"initial_validation": {
"target_rel": 0.20227570505264494,
"reference_rel": 0.20227570505264494,
"windows": [
{
"first_row": 0,
"rows": 1024,
"reference_sse": 59633.68359375,
"reference_energy": 2000581.5
},
{
"first_row": 1024,
"rows": 1024,
"reference_sse": 80431.3515625,
"reference_energy": 1939419.75
},
{
"first_row": 4096,
"rows": 1024,
"reference_sse": 322369.375,
"reference_energy": 3524174.5
},
{
"first_row": 5120,
"rows": 1024,
"reference_sse": 67250.7890625,
"reference_energy": 1945419.875
},
{
"first_row": 8192,
"rows": 1024,
"reference_sse": 162280.640625,
"reference_energy": 1955614.375
},
{
"first_row": 9216,
"rows": 1024,
"reference_sse": 70590.7734375,
"reference_energy": 5132028.5
},
{
"first_row": 12288,
"rows": 1024,
"reference_sse": 68240.234375,
"reference_energy": 1956487.125
},
{
"first_row": 13312,
"rows": 1024,
"reference_sse": 57198.9140625,
"reference_energy": 2042521.5
},
{
"first_row": 16384,
"rows": 1024,
"reference_sse": 58534.6796875,
"reference_energy": 2019550.0
},
{
"first_row": 17408,
"rows": 1024,
"reference_sse": 59036.06640625,
"reference_energy": 2066729.5
},
{
"first_row": 20480,
"rows": 1024,
"reference_sse": 57641.1953125,
"reference_energy": 2055120.875
},
{
"first_row": 21504,
"rows": 1024,
"reference_sse": 94883.1484375,
"reference_energy": 3464950.25
},
{
"first_row": 24576,
"rows": 1024,
"reference_sse": 74537.3046875,
"reference_energy": 2045602.5
},
{
"first_row": 25600,
"rows": 1024,
"reference_sse": 48334.5703125,
"reference_energy": 2046166.75
},
{
"first_row": 28672,
"rows": 1024,
"reference_sse": 51436.2578125,
"reference_energy": 1924115.25
},
{
"first_row": 29696,
"rows": 1024,
"reference_sse": 228344.765625,
"reference_energy": 2027092.0
}
]
},
"final_validation": {
"target_rel": 0.20210293694613474,
"reference_rel": 0.20210293694613474,
"windows": [
{
"first_row": 0,
"rows": 1024,
"reference_sse": 59520.3203125,
"reference_energy": 2000581.5
},
{
"first_row": 1024,
"rows": 1024,
"reference_sse": 80251.0078125,
"reference_energy": 1939419.75
},
{
"first_row": 4096,
"rows": 1024,
"reference_sse": 322016.1875,
"reference_energy": 3524174.5
},
{
"first_row": 5120,
"rows": 1024,
"reference_sse": 67109.3125,
"reference_energy": 1945419.875
},
{
"first_row": 8192,
"rows": 1024,
"reference_sse": 162099.6875,
"reference_energy": 1955614.375
},
{
"first_row": 9216,
"rows": 1024,
"reference_sse": 70321.4765625,
"reference_energy": 5132028.5
},
{
"first_row": 12288,
"rows": 1024,
"reference_sse": 68073.859375,
"reference_energy": 1956487.125
},
{
"first_row": 13312,
"rows": 1024,
"reference_sse": 57094.390625,
"reference_energy": 2042521.5
},
{
"first_row": 16384,
"rows": 1024,
"reference_sse": 58408.171875,
"reference_energy": 2019550.0
},
{
"first_row": 17408,
"rows": 1024,
"reference_sse": 58935.265625,
"reference_energy": 2066729.5
},
{
"first_row": 20480,
"rows": 1024,
"reference_sse": 57526.4375,
"reference_energy": 2055120.875
},
{
"first_row": 21504,
"rows": 1024,
"reference_sse": 94456.46875,
"reference_energy": 3464950.25
},
{
"first_row": 24576,
"rows": 1024,
"reference_sse": 74429.5625,
"reference_energy": 2045602.5
},
{
"first_row": 25600,
"rows": 1024,
"reference_sse": 48255.9296875,
"reference_energy": 2046166.75
},
{
"first_row": 28672,
"rows": 1024,
"reference_sse": 51290.8359375,
"reference_energy": 1924115.25
},
{
"first_row": 29696,
"rows": 1024,
"reference_sse": 228289.84375,
"reference_energy": 2027092.0
}
]
},
"initial_reasoning_validation": null,
"final_reasoning_validation": null,
"validation_distribution": "reasoning only",
"serialized_validation": {
"target_rel": 0.20210293694613474,
"reference_rel": 0.20210293694613474
},
"development_audit": {
"target_rel": 0.17275675760047116,
"reference_rel": 0.17275675760047116,
"windows": [
{
"first_row": 2048,
"rows": 1024,
"reference_sse": 61017.40625,
"reference_energy": 1979178.0
},
{
"first_row": 3072,
"rows": 1024,
"reference_sse": 72327.1953125,
"reference_energy": 2078790.25
},
{
"first_row": 6144,
"rows": 1024,
"reference_sse": 54185.19140625,
"reference_energy": 1993549.0
},
{
"first_row": 7168,
"rows": 1024,
"reference_sse": 64342.86328125,
"reference_energy": 5075389.0
},
{
"first_row": 10240,
"rows": 1024,
"reference_sse": 81875.71875,
"reference_energy": 3524613.0
},
{
"first_row": 11264,
"rows": 1024,
"reference_sse": 52481.875,
"reference_energy": 2043313.375
},
{
"first_row": 14336,
"rows": 1024,
"reference_sse": 154725.46875,
"reference_energy": 3552758.5
},
{
"first_row": 15360,
"rows": 1024,
"reference_sse": 47327.828125,
"reference_energy": 2062143.125
},
{
"first_row": 18432,
"rows": 1024,
"reference_sse": 66431.96875,
"reference_energy": 1929567.875
},
{
"first_row": 19456,
"rows": 1024,
"reference_sse": 64307.60546875,
"reference_energy": 2011222.75
},
{
"first_row": 22528,
"rows": 1024,
"reference_sse": 59424.109375,
"reference_energy": 1956750.75
},
{
"first_row": 23552,
"rows": 1024,
"reference_sse": 151521.1875,
"reference_energy": 1982972.75
},
{
"first_row": 26624,
"rows": 1024,
"reference_sse": 71733.234375,
"reference_energy": 1997533.0
},
{
"first_row": 27648,
"rows": 1024,
"reference_sse": 66455.5546875,
"reference_energy": 3520451.25
},
{
"first_row": 30720,
"rows": 1024,
"reference_sse": 54924.75,
"reference_energy": 1937144.5
},
{
"first_row": 31744,
"rows": 1024,
"reference_sse": 59220.1015625,
"reference_energy": 1969504.0
}
]
},
"initial_development_audit": {
"target_rel": 0.17290494144443153,
"reference_rel": 0.17290494144443153,
"windows": [
{
"first_row": 2048,
"rows": 1024,
"reference_sse": 61149.48828125,
"reference_energy": 1979178.0
},
{
"first_row": 3072,
"rows": 1024,
"reference_sse": 72448.0234375,
"reference_energy": 2078790.25
},
{
"first_row": 6144,
"rows": 1024,
"reference_sse": 54315.0703125,
"reference_energy": 1993549.0
},
{
"first_row": 7168,
"rows": 1024,
"reference_sse": 64497.25,
"reference_energy": 5075389.0
},
{
"first_row": 10240,
"rows": 1024,
"reference_sse": 82086.671875,
"reference_energy": 3524613.0
},
{
"first_row": 11264,
"rows": 1024,
"reference_sse": 52624.734375,
"reference_energy": 2043313.375
},
{
"first_row": 14336,
"rows": 1024,
"reference_sse": 154878.953125,
"reference_energy": 3552758.5
},
{
"first_row": 15360,
"rows": 1024,
"reference_sse": 47423.15234375,
"reference_energy": 2062143.125
},
{
"first_row": 18432,
"rows": 1024,
"reference_sse": 66540.0703125,
"reference_energy": 1929567.875
},
{
"first_row": 19456,
"rows": 1024,
"reference_sse": 64455.03125,
"reference_energy": 2011222.75
},
{
"first_row": 22528,
"rows": 1024,
"reference_sse": 59523.78515625,
"reference_energy": 1956750.75
},
{
"first_row": 23552,
"rows": 1024,
"reference_sse": 151566.34375,
"reference_energy": 1982972.75
},
{
"first_row": 26624,
"rows": 1024,
"reference_sse": 71841.4453125,
"reference_energy": 1997533.0
},
{
"first_row": 27648,
"rows": 1024,
"reference_sse": 66588.0078125,
"reference_energy": 3520451.25
},
{
"first_row": 30720,
"rows": 1024,
"reference_sse": 55037.609375,
"reference_energy": 1937144.5
},
{
"first_row": 31744,
"rows": 1024,
"reference_sse": 59355.5546875,
"reference_energy": 1969504.0
}
]
},
"best_step": 35,
"steps_run": 60,
"history": [
{
"step": 5,
"validation": {
"target_rel": 0.20221232905145298,
"reference_rel": 0.20221232905145298
},
"reasoning_validation": null,
"training_probe_rel": 0.9728345308771068,
"batch_loss": 0.9120448678731918,
"best_step": 5,
"overfit_checks": 0,
"seconds": 55.40898847579956,
"adaptive_schedule": {
"best": 0.20221232905145298,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 5,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 0,
"codebook_values": 1425408,
"scales_changed": 8588582,
"scale_values": 51314688
}
},
{
"step": 10,
"validation": {
"target_rel": 0.2021785889916135,
"reference_rel": 0.2021785889916135
},
"reasoning_validation": null,
"training_probe_rel": 0.971972986580314,
"batch_loss": 0.9077726006507874,
"best_step": 10,
"overfit_checks": 0,
"seconds": 107.47525405883789,
"adaptive_schedule": {
"best": 0.2021785889916135,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 10,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 19357,
"codebook_values": 1425408,
"scales_changed": 18154941,
"scale_values": 51314688
}
},
{
"step": 15,
"validation": {
"target_rel": 0.20215371621311168,
"reference_rel": 0.20215371621311168
},
"reasoning_validation": null,
"training_probe_rel": 0.9713020010898051,
"batch_loss": 0.9612409323453903,
"best_step": 15,
"overfit_checks": 0,
"seconds": 162.7464816570282,
"adaptive_schedule": {
"best": 0.20215371621311168,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 15,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 3099,
"codebook_values": 1425408,
"scales_changed": 23543597,
"scale_values": 51314688
}
},
{
"step": 20,
"validation": {
"target_rel": 0.20213030071848295,
"reference_rel": 0.20213030071848295
},
"reasoning_validation": null,
"training_probe_rel": 0.9708750604948976,
"batch_loss": 0.9377660155296326,
"best_step": 20,
"overfit_checks": 0,
"seconds": 234.0874376296997,
"adaptive_schedule": {
"best": 0.20213030071848295,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 20,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 3198,
"codebook_values": 1425408,
"scales_changed": 27045347,
"scale_values": 51314688
}
},
{
"step": 25,
"validation": {
"target_rel": 0.2021173472331997,
"reference_rel": 0.2021173472331997
},
"reasoning_validation": null,
"training_probe_rel": 0.9705821134193601,
"batch_loss": 0.9458546191453934,
"best_step": 25,
"overfit_checks": 0,
"seconds": 290.8809871673584,
"adaptive_schedule": {
"best": 0.2021173472331997,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 25,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 2872,
"codebook_values": 1425408,
"scales_changed": 29483693,
"scale_values": 51314688
}
},
{
"step": 30,
"validation": {
"target_rel": 0.2021112173753633,
"reference_rel": 0.2021112173753633
},
"reasoning_validation": null,
"training_probe_rel": 0.9703887773645942,
"batch_loss": 0.935170903801918,
"best_step": 30,
"overfit_checks": 0,
"seconds": 349.4162108898163,
"adaptive_schedule": {
"best": 0.2021112173753633,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 30,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 2859,
"codebook_values": 1425408,
"scales_changed": 31263562,
"scale_values": 51314688
}
},
{
"step": 35,
"validation": {
"target_rel": 0.20210293694613474,
"reference_rel": 0.20210293694613474
},
"reasoning_validation": null,
"training_probe_rel": 0.9702341957221163,
"batch_loss": 0.9798422455787659,
"best_step": 35,
"overfit_checks": 0,
"seconds": 399.80804920196533,
"adaptive_schedule": {
"best": 0.20210293694613474,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 35,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 2336,
"codebook_values": 1425408,
"scales_changed": 32618897,
"scale_values": 51314688
}
},
{
"step": 40,
"validation": {
"target_rel": 0.20211535505612807,
"reference_rel": 0.20211535505612807
},
"reasoning_validation": null,
"training_probe_rel": 0.9700557213360986,
"batch_loss": 0.8994913846254349,
"best_step": 35,
"overfit_checks": 0,
"seconds": 470.49743485450745,
"adaptive_schedule": {
"best": 0.20210293694613474,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 35,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 2317,
"codebook_values": 1425408,
"scales_changed": 33685985,
"scale_values": 51314688
}
},
{
"step": 45,
"validation": {
"target_rel": 0.20212493298204956,
"reference_rel": 0.20212493298204956
},
"reasoning_validation": null,
"training_probe_rel": 0.9699300833766342,
"batch_loss": 0.8946956247091293,
"best_step": 35,
"overfit_checks": 0,
"seconds": 520.6946864128113,
"adaptive_schedule": {
"best": 0.20210293694613474,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 35,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 2227,
"codebook_values": 1425408,
"scales_changed": 34543560,
"scale_values": 51314688
}
},
{
"step": 50,
"validation": {
"target_rel": 0.2021209776366218,
"reference_rel": 0.2021209776366218
},
"reasoning_validation": null,
"training_probe_rel": 0.9698820868103767,
"batch_loss": 0.9467693865299225,
"best_step": 35,
"overfit_checks": 0,
"seconds": 578.9339537620544,
"adaptive_schedule": {
"best": 0.20210293694613474,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 35,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 1336,
"codebook_values": 1425408,
"scales_changed": 34714761,
"scale_values": 51314688
}
},
{
"step": 55,
"validation": {
"target_rel": 0.20211545816139986,
"reference_rel": 0.20211545816139986
},
"reasoning_validation": null,
"training_probe_rel": 0.9698829842263352,
"batch_loss": 1.0004937499761581,
"best_step": 35,
"overfit_checks": 0,
"seconds": 629.9127867221832,
"adaptive_schedule": {
"best": 0.20210293694613474,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 35,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 1374,
"codebook_values": 1425408,
"scales_changed": 34859751,
"scale_values": 51314688
}
},
{
"step": 60,
"validation": {
"target_rel": 0.20221304689622666,
"reference_rel": 0.20221304689622666
},
"reasoning_validation": null,
"training_probe_rel": 0.9697254329355303,
"batch_loss": 0.9330670237541199,
"best_step": 35,
"overfit_checks": 0,
"seconds": 694.2890436649323,
"adaptive_schedule": {
"best": 0.20210293694613474,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 35,
"early_drop": false
},
"adaptive_stop": true,
"quantized_movement": {
"codebook_changed": 1426,
"codebook_values": 1425408,
"scales_changed": 34985559,
"scale_values": 51314688
}
}
],
"index_reassignments": [
{
"method": "expert_parallel_output_gradient_prefix_backtracking_v1",
"proposals": 332,
"accepted": 292,
"changed_groups": 206744,
"training_before": 824457.0,
"training_after": 823630.25,
"acceptance": "proposal and separate training check batches",
"step": 20
},
{
"method": "expert_parallel_output_gradient_prefix_backtracking_v1",
"proposals": 334,
"accepted": 282,
"changed_groups": 216494,
"training_before": 735006.8125,
"training_after": 734268.0625,
"acceptance": "proposal and separate training check batches",
"step": 40
},
{
"method": "expert_parallel_output_gradient_prefix_backtracking_v1",
"proposals": 332,
"accepted": 291,
"changed_groups": 213329,
"training_before": 756520.0,
"training_after": 755339.75,
"acceptance": "proposal and separate training check batches",
"step": 60
}
],
"coverage_threshold": 256,
"checkpoint_selection": "held-out original reference trajectory for both arms",
"world_size": 8,
"microbatch": 65536,
"proposal_communication": "sparse routed rows",
"parallelism": "expert-sharded coupled output",
"arithmetic": "fp4_planes4_fp16_boundaries_v1",
"scope": "text-only, cached prompt partitions; historical audit is development only",
"train_rows": 18001846,
"validation_rows": 16384,
"audit_rows": 16384,
"complete": true
}