jarrelscy's picture
Publish GLM-5.3 NVFP4 / ARVQ v2 same_input fp16-block hybrid: non-cold structure seed (part 2)
868041e verified
Raw History Blame
22.6 kB
{
"layer": 37,
"target": "reference",
"recipe": "full_corpus_sequence_pass_v1",
"block_scale_dtype": "fp8_e4m3",
"batch_tokens": 262144,
"sequence_order_sha256": "edebb22a5f7b34a6cbfa59c58aab9f111651d6ce0926212b5237dea3ef0ba485",
"training_sequence_count": 17580,
"training_passes": 0.8737237281109949,
"training_tokens_seen": 15728640,
"adaptive_schedule": {
"best": 0.17073380692221246,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 35,
"early_drop": false
},
"refinement": null,
"lr_decay_after": 45,
"lr_decay_factor": 0.25,
"export_only": false,
"training_output_export": "fixed validation/audit outputs only; full training trajectory export not performed",
"codebook_lr": 0.048,
"scale_lr": 0.032,
"quantized_movement": {
"codebook_changed": 3632,
"codebook_values": 1507328,
"scales_changed": 35177900,
"scale_values": 54263808
},
"initial_validation": {
"target_rel": 0.17096866877600614,
"reference_rel": 0.17096866877600614,
"windows": [
{
"first_row": 0,
"rows": 1024,
"reference_sse": 26381.92578125,
"reference_energy": 1732637.5
},
{
"first_row": 1024,
"rows": 1024,
"reference_sse": 41476.46875,
"reference_energy": 1683678.75
},
{
"first_row": 4096,
"rows": 1024,
"reference_sse": 273111.25,
"reference_energy": 3254165.75
},
{
"first_row": 5120,
"rows": 1024,
"reference_sse": 28472.05859375,
"reference_energy": 1686845.25
},
{
"first_row": 8192,
"rows": 1024,
"reference_sse": 136671.75,
"reference_energy": 1666810.5
},
{
"first_row": 9216,
"rows": 1024,
"reference_sse": 33516.1953125,
"reference_energy": 4815264.5
},
{
"first_row": 12288,
"rows": 1024,
"reference_sse": 31507.20703125,
"reference_energy": 1699171.25
},
{
"first_row": 13312,
"rows": 1024,
"reference_sse": 22355.65625,
"reference_energy": 1766444.875
},
{
"first_row": 16384,
"rows": 1024,
"reference_sse": 26942.126953125,
"reference_energy": 1749859.125
},
{
"first_row": 17408,
"rows": 1024,
"reference_sse": 27513.708984375,
"reference_energy": 1804149.75
},
{
"first_row": 20480,
"rows": 1024,
"reference_sse": 26745.81640625,
"reference_energy": 1797582.5
},
{
"first_row": 21504,
"rows": 1024,
"reference_sse": 43973.234375,
"reference_energy": 3192390.25
},
{
"first_row": 24576,
"rows": 1024,
"reference_sse": 30615.900390625,
"reference_energy": 1761519.5
},
{
"first_row": 25600,
"rows": 1024,
"reference_sse": 22030.84375,
"reference_energy": 1780874.125
},
{
"first_row": 28672,
"rows": 1024,
"reference_sse": 25173.58984375,
"reference_energy": 1660917.25
},
{
"first_row": 29696,
"rows": 1024,
"reference_sse": 191892.03125,
"reference_energy": 1761240.375
}
]
},
"final_validation": {
"target_rel": 0.17073380692221246,
"reference_rel": 0.17073380692221246,
"windows": [
{
"first_row": 0,
"rows": 1024,
"reference_sse": 26361.880859375,
"reference_energy": 1732637.5
},
{
"first_row": 1024,
"rows": 1024,
"reference_sse": 41306.23046875,
"reference_energy": 1683678.75
},
{
"first_row": 4096,
"rows": 1024,
"reference_sse": 272244.15625,
"reference_energy": 3254165.75
},
{
"first_row": 5120,
"rows": 1024,
"reference_sse": 28445.11328125,
"reference_energy": 1686845.25
},
{
"first_row": 8192,
"rows": 1024,
"reference_sse": 136219.953125,
"reference_energy": 1666810.5
},
{
"first_row": 9216,
"rows": 1024,
"reference_sse": 33436.125,
"reference_energy": 4815264.5
},
{
"first_row": 12288,
"rows": 1024,
"reference_sse": 31490.86328125,
"reference_energy": 1699171.25
},
{
"first_row": 13312,
"rows": 1024,
"reference_sse": 22352.12109375,
"reference_energy": 1766444.875
},
{
"first_row": 16384,
"rows": 1024,
"reference_sse": 26908.0703125,
"reference_energy": 1749859.125
},
{
"first_row": 17408,
"rows": 1024,
"reference_sse": 27487.5,
"reference_energy": 1804149.75
},
{
"first_row": 20480,
"rows": 1024,
"reference_sse": 26703.109375,
"reference_energy": 1797582.5
},
{
"first_row": 21504,
"rows": 1024,
"reference_sse": 43676.77734375,
"reference_energy": 3192390.25
},
{
"first_row": 24576,
"rows": 1024,
"reference_sse": 30584.927734375,
"reference_energy": 1761519.5
},
{
"first_row": 25600,
"rows": 1024,
"reference_sse": 22010.078125,
"reference_energy": 1780874.125
},
{
"first_row": 28672,
"rows": 1024,
"reference_sse": 25152.34765625,
"reference_energy": 1660917.25
},
{
"first_row": 29696,
"rows": 1024,
"reference_sse": 191286.875,
"reference_energy": 1761240.375
}
]
},
"initial_reasoning_validation": null,
"final_reasoning_validation": null,
"validation_distribution": "reasoning only",
"serialized_validation": {
"target_rel": 0.17073380692221246,
"reference_rel": 0.17073380692221246
},
"development_audit": {
"target_rel": 0.1331582651037434,
"reference_rel": 0.1331582651037434,
"windows": [
{
"first_row": 2048,
"rows": 1024,
"reference_sse": 27489.5625,
"reference_energy": 1707031.375
},
{
"first_row": 3072,
"rows": 1024,
"reference_sse": 31765.30078125,
"reference_energy": 1822428.0
},
{
"first_row": 6144,
"rows": 1024,
"reference_sse": 25330.875,
"reference_energy": 1737341.875
},
{
"first_row": 7168,
"rows": 1024,
"reference_sse": 30012.658203125,
"reference_energy": 4777391.0
},
{
"first_row": 10240,
"rows": 1024,
"reference_sse": 36790.359375,
"reference_energy": 3257501.0
},
{
"first_row": 11264,
"rows": 1024,
"reference_sse": 24215.75390625,
"reference_energy": 1779650.375
},
{
"first_row": 14336,
"rows": 1024,
"reference_sse": 113828.140625,
"reference_energy": 3288287.0
},
{
"first_row": 15360,
"rows": 1024,
"reference_sse": 20391.619140625,
"reference_energy": 1805148.125
},
{
"first_row": 18432,
"rows": 1024,
"reference_sse": 27984.142578125,
"reference_energy": 1642787.25
},
{
"first_row": 19456,
"rows": 1024,
"reference_sse": 28626.677734375,
"reference_energy": 1756180.75
},
{
"first_row": 22528,
"rows": 1024,
"reference_sse": 26075.75390625,
"reference_energy": 1700101.25
},
{
"first_row": 23552,
"rows": 1024,
"reference_sse": 122137.734375,
"reference_energy": 1709515.625
},
{
"first_row": 26624,
"rows": 1024,
"reference_sse": 30647.625,
"reference_energy": 1726092.625
},
{
"first_row": 27648,
"rows": 1024,
"reference_sse": 30661.375,
"reference_energy": 3264232.75
},
{
"first_row": 30720,
"rows": 1024,
"reference_sse": 23859.82421875,
"reference_energy": 1660031.5
},
{
"first_row": 31744,
"rows": 1024,
"reference_sse": 27033.580078125,
"reference_energy": 1719424.5
}
]
},
"initial_development_audit": {
"target_rel": 0.1332898631973475,
"reference_rel": 0.1332898631973475,
"windows": [
{
"first_row": 2048,
"rows": 1024,
"reference_sse": 27496.2890625,
"reference_energy": 1707031.375
},
{
"first_row": 3072,
"rows": 1024,
"reference_sse": 31766.28515625,
"reference_energy": 1822428.0
},
{
"first_row": 6144,
"rows": 1024,
"reference_sse": 25339.015625,
"reference_energy": 1737341.875
},
{
"first_row": 7168,
"rows": 1024,
"reference_sse": 30133.2109375,
"reference_energy": 4777391.0
},
{
"first_row": 10240,
"rows": 1024,
"reference_sse": 36891.78515625,
"reference_energy": 3257501.0
},
{
"first_row": 11264,
"rows": 1024,
"reference_sse": 24274.390625,
"reference_energy": 1779650.375
},
{
"first_row": 14336,
"rows": 1024,
"reference_sse": 113887.2578125,
"reference_energy": 3288287.0
},
{
"first_row": 15360,
"rows": 1024,
"reference_sse": 20414.892578125,
"reference_energy": 1805148.125
},
{
"first_row": 18432,
"rows": 1024,
"reference_sse": 28002.29296875,
"reference_energy": 1642787.25
},
{
"first_row": 19456,
"rows": 1024,
"reference_sse": 28668.587890625,
"reference_energy": 1756180.75
},
{
"first_row": 22528,
"rows": 1024,
"reference_sse": 26144.734375,
"reference_energy": 1700101.25
},
{
"first_row": 23552,
"rows": 1024,
"reference_sse": 122590.21875,
"reference_energy": 1709515.625
},
{
"first_row": 26624,
"rows": 1024,
"reference_sse": 30661.634765625,
"reference_energy": 1726092.625
},
{
"first_row": 27648,
"rows": 1024,
"reference_sse": 30780.140625,
"reference_energy": 3264232.75
},
{
"first_row": 30720,
"rows": 1024,
"reference_sse": 23914.81640625,
"reference_energy": 1660031.5
},
{
"first_row": 31744,
"rows": 1024,
"reference_sse": 27125.0546875,
"reference_energy": 1719424.5
}
]
},
"best_step": 35,
"steps_run": 60,
"history": [
{
"step": 5,
"validation": {
"target_rel": 0.17090702723695667,
"reference_rel": 0.17090702723695667
},
"reasoning_validation": null,
"training_probe_rel": 0.948087968523702,
"batch_loss": 0.8801274597644806,
"best_step": 5,
"overfit_checks": 0,
"seconds": 49.25497269630432,
"adaptive_schedule": {
"best": 0.17090702723695667,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 5,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 0,
"codebook_values": 1507328,
"scales_changed": 10445268,
"scale_values": 54263808
}
},
{
"step": 10,
"validation": {
"target_rel": 0.17091387861510668,
"reference_rel": 0.17091387861510668
},
"reasoning_validation": null,
"training_probe_rel": 0.9476216087055943,
"batch_loss": 0.8769833743572235,
"best_step": 5,
"overfit_checks": 0,
"seconds": 98.78876185417175,
"adaptive_schedule": {
"best": 0.17090702723695667,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 5,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 24218,
"codebook_values": 1507328,
"scales_changed": 20592551,
"scale_values": 54263808
}
},
{
"step": 15,
"validation": {
"target_rel": 0.17086604639133351,
"reference_rel": 0.17086604639133351
},
"reasoning_validation": null,
"training_probe_rel": 0.946215528076647,
"batch_loss": 0.967778891324997,
"best_step": 15,
"overfit_checks": 0,
"seconds": 151.7948112487793,
"adaptive_schedule": {
"best": 0.17086604639133351,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 15,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 4189,
"codebook_values": 1507328,
"scales_changed": 26092710,
"scale_values": 54263808
}
},
{
"step": 20,
"validation": {
"target_rel": 0.17078762890289342,
"reference_rel": 0.17078762890289342
},
"reasoning_validation": null,
"training_probe_rel": 0.9460301821985658,
"batch_loss": 0.918610692024231,
"best_step": 20,
"overfit_checks": 0,
"seconds": 214.7258861064911,
"adaptive_schedule": {
"best": 0.17078762890289342,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 20,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 4777,
"codebook_values": 1507328,
"scales_changed": 29632720,
"scale_values": 54263808
}
},
{
"step": 25,
"validation": {
"target_rel": 0.17075548469516996,
"reference_rel": 0.17075548469516996
},
"reasoning_validation": null,
"training_probe_rel": 0.9456897029773403,
"batch_loss": 0.9156636744737625,
"best_step": 25,
"overfit_checks": 0,
"seconds": 268.3811161518097,
"adaptive_schedule": {
"best": 0.17075548469516996,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 25,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 4617,
"codebook_values": 1507328,
"scales_changed": 32071742,
"scale_values": 54263808
}
},
{
"step": 30,
"validation": {
"target_rel": 0.17075140070219522,
"reference_rel": 0.17075140070219522
},
"reasoning_validation": null,
"training_probe_rel": 0.9454739712134431,
"batch_loss": 0.9078531265258789,
"best_step": 30,
"overfit_checks": 0,
"seconds": 318.2578799724579,
"adaptive_schedule": {
"best": 0.17075140070219522,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 30,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 4172,
"codebook_values": 1507328,
"scales_changed": 33839817,
"scale_values": 54263808
}
},
{
"step": 35,
"validation": {
"target_rel": 0.17073380692221246,
"reference_rel": 0.17073380692221246
},
"reasoning_validation": null,
"training_probe_rel": 0.9453546911235419,
"batch_loss": 0.9876965135335922,
"best_step": 35,
"overfit_checks": 0,
"seconds": 373.3670885562897,
"adaptive_schedule": {
"best": 0.17073380692221246,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 35,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 3632,
"codebook_values": 1507328,
"scales_changed": 35177900,
"scale_values": 54263808
}
},
{
"step": 40,
"validation": {
"target_rel": 0.17081475079745237,
"reference_rel": 0.17081475079745237
},
"reasoning_validation": null,
"training_probe_rel": 0.9449445090906345,
"batch_loss": 0.8515045493841171,
"best_step": 35,
"overfit_checks": 0,
"seconds": 439.10769844055176,
"adaptive_schedule": {
"best": 0.17073380692221246,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 35,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 3455,
"codebook_values": 1507328,
"scales_changed": 36232389,
"scale_values": 54263808
}
},
{
"step": 45,
"validation": {
"target_rel": 0.1707665684568324,
"reference_rel": 0.1707665684568324
},
"reasoning_validation": null,
"training_probe_rel": 0.9450725225639061,
"batch_loss": 0.8545539230108261,
"best_step": 35,
"overfit_checks": 0,
"seconds": 494.0295286178589,
"adaptive_schedule": {
"best": 0.17073380692221246,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 35,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 3522,
"codebook_values": 1507328,
"scales_changed": 37073458,
"scale_values": 54263808
}
},
{
"step": 50,
"validation": {
"target_rel": 0.17073502687746445,
"reference_rel": 0.17073502687746445
},
"reasoning_validation": null,
"training_probe_rel": 0.9450121367960164,
"batch_loss": 0.9334661662578583,
"best_step": 35,
"overfit_checks": 0,
"seconds": 545.8804841041565,
"adaptive_schedule": {
"best": 0.17073380692221246,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 35,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 1872,
"codebook_values": 1507328,
"scales_changed": 37239835,
"scale_values": 54263808
}
},
{
"step": 55,
"validation": {
"target_rel": 0.17074000471877798,
"reference_rel": 0.17074000471877798
},
"reasoning_validation": null,
"training_probe_rel": 0.9449904014427146,
"batch_loss": 1.0141641199588776,
"best_step": 35,
"overfit_checks": 0,
"seconds": 600.7391157150269,
"adaptive_schedule": {
"best": 0.17073380692221246,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 35,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 2251,
"codebook_values": 1507328,
"scales_changed": 37373527,
"scale_values": 54263808
}
},
{
"step": 60,
"validation": {
"target_rel": 0.17076549688093043,
"reference_rel": 0.17076549688093043
},
"reasoning_validation": null,
"training_probe_rel": 0.9448911377189974,
"batch_loss": 0.9167208969593048,
"best_step": 35,
"overfit_checks": 0,
"seconds": 666.040885925293,
"adaptive_schedule": {
"best": 0.17073380692221246,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 35,
"early_drop": false
},
"adaptive_stop": true,
"quantized_movement": {
"codebook_changed": 2270,
"codebook_values": 1507328,
"scales_changed": 37491071,
"scale_values": 54263808
}
}
],
"index_reassignments": [
{
"method": "expert_parallel_output_gradient_prefix_backtracking_v1",
"proposals": 364,
"accepted": 319,
"changed_groups": 290344,
"training_before": 480879.25,
"training_after": 480490.0,
"acceptance": "proposal and separate training check batches",
"step": 20
},
{
"method": "expert_parallel_output_gradient_prefix_backtracking_v1",
"proposals": 364,
"accepted": 308,
"changed_groups": 316719,
"training_before": 388256.125,
"training_after": 387536.65625,
"acceptance": "proposal and separate training check batches",
"step": 40
},
{
"method": "expert_parallel_output_gradient_prefix_backtracking_v1",
"proposals": 362,
"accepted": 300,
"changed_groups": 309672,
"training_before": 414220.8125,
"training_after": 413936.1875,
"acceptance": "proposal and separate training check batches",
"step": 60
}
],
"coverage_threshold": 256,
"checkpoint_selection": "held-out original reference trajectory for both arms",
"world_size": 8,
"microbatch": 65536,
"proposal_communication": "sparse routed rows",
"parallelism": "expert-sharded coupled output",
"arithmetic": "fp4_planes4_fp16_boundaries_v1",
"scope": "text-only, cached prompt partitions; historical audit is development only",
"train_rows": 18001846,
"validation_rows": 16384,
"audit_rows": 16384,
"complete": true
}