jarrelscy's picture
Publish GLM-5.3 NVFP4 / ARVQ v2 same_input fp16-block hybrid: non-cold structure seed (part 2)
868041e verified
Raw History Blame
22.6 kB
{
"layer": 38,
"target": "reference",
"recipe": "full_corpus_sequence_pass_v1",
"block_scale_dtype": "fp8_e4m3",
"batch_tokens": 262144,
"sequence_order_sha256": "edebb22a5f7b34a6cbfa59c58aab9f111651d6ce0926212b5237dea3ef0ba485",
"training_sequence_count": 17580,
"training_passes": 0.8737237281109949,
"training_tokens_seen": 15728640,
"adaptive_schedule": {
"best": 0.17944146946888725,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 40,
"early_drop": false
},
"refinement": null,
"lr_decay_after": 45,
"lr_decay_factor": 0.25,
"export_only": false,
"training_output_export": "fixed validation/audit outputs only; full training trajectory export not performed",
"codebook_lr": 0.048,
"scale_lr": 0.032,
"quantized_movement": {
"codebook_changed": 3131,
"codebook_values": 1376256,
"scales_changed": 32309979,
"scale_values": 49545216
},
"initial_validation": {
"target_rel": 0.17960681601178594,
"reference_rel": 0.17960681601178594,
"windows": [
{
"first_row": 0,
"rows": 1024,
"reference_sse": 33894.703125,
"reference_energy": 1785723.375
},
{
"first_row": 1024,
"rows": 1024,
"reference_sse": 50507.2109375,
"reference_energy": 1733528.0
},
{
"first_row": 4096,
"rows": 1024,
"reference_sse": 284606.21875,
"reference_energy": 3303205.0
},
{
"first_row": 5120,
"rows": 1024,
"reference_sse": 37341.09375,
"reference_energy": 1740884.0
},
{
"first_row": 8192,
"rows": 1024,
"reference_sse": 142648.328125,
"reference_energy": 1724684.5
},
{
"first_row": 9216,
"rows": 1024,
"reference_sse": 41814.6484375,
"reference_energy": 4872984.5
},
{
"first_row": 12288,
"rows": 1024,
"reference_sse": 40226.625,
"reference_energy": 1751318.875
},
{
"first_row": 13312,
"rows": 1024,
"reference_sse": 29908.46875,
"reference_energy": 1823804.5
},
{
"first_row": 16384,
"rows": 1024,
"reference_sse": 33635.8984375,
"reference_energy": 1796564.625
},
{
"first_row": 17408,
"rows": 1024,
"reference_sse": 34560.6015625,
"reference_energy": 1858712.125
},
{
"first_row": 20480,
"rows": 1024,
"reference_sse": 33950.1015625,
"reference_energy": 1849871.75
},
{
"first_row": 21504,
"rows": 1024,
"reference_sse": 54673.0546875,
"reference_energy": 3243022.5
},
{
"first_row": 24576,
"rows": 1024,
"reference_sse": 41328.8125,
"reference_energy": 1822806.25
},
{
"first_row": 25600,
"rows": 1024,
"reference_sse": 28009.115234375,
"reference_energy": 1837471.0
},
{
"first_row": 28672,
"rows": 1024,
"reference_sse": 31510.181640625,
"reference_energy": 1715521.75
},
{
"first_row": 29696,
"rows": 1024,
"reference_sse": 200080.65625,
"reference_energy": 1818879.875
}
]
},
"final_validation": {
"target_rel": 0.17944146946888725,
"reference_rel": 0.17944146946888725,
"windows": [
{
"first_row": 0,
"rows": 1024,
"reference_sse": 33847.0703125,
"reference_energy": 1785723.375
},
{
"first_row": 1024,
"rows": 1024,
"reference_sse": 50358.1640625,
"reference_energy": 1733528.0
},
{
"first_row": 4096,
"rows": 1024,
"reference_sse": 284150.84375,
"reference_energy": 3303205.0
},
{
"first_row": 5120,
"rows": 1024,
"reference_sse": 37293.6171875,
"reference_energy": 1740884.0
},
{
"first_row": 8192,
"rows": 1024,
"reference_sse": 142405.21875,
"reference_energy": 1724684.5
},
{
"first_row": 9216,
"rows": 1024,
"reference_sse": 41675.0859375,
"reference_energy": 4872984.5
},
{
"first_row": 12288,
"rows": 1024,
"reference_sse": 40164.7265625,
"reference_energy": 1751318.875
},
{
"first_row": 13312,
"rows": 1024,
"reference_sse": 29872.09765625,
"reference_energy": 1823804.5
},
{
"first_row": 16384,
"rows": 1024,
"reference_sse": 33581.9296875,
"reference_energy": 1796564.625
},
{
"first_row": 17408,
"rows": 1024,
"reference_sse": 34493.39453125,
"reference_energy": 1858712.125
},
{
"first_row": 20480,
"rows": 1024,
"reference_sse": 33876.91015625,
"reference_energy": 1849871.75
},
{
"first_row": 21504,
"rows": 1024,
"reference_sse": 54415.375,
"reference_energy": 3243022.5
},
{
"first_row": 24576,
"rows": 1024,
"reference_sse": 41266.671875,
"reference_energy": 1822806.25
},
{
"first_row": 25600,
"rows": 1024,
"reference_sse": 27954.63671875,
"reference_energy": 1837471.0
},
{
"first_row": 28672,
"rows": 1024,
"reference_sse": 31449.42578125,
"reference_energy": 1715521.75
},
{
"first_row": 29696,
"rows": 1024,
"reference_sse": 199831.75,
"reference_energy": 1818879.875
}
]
},
"initial_reasoning_validation": null,
"final_reasoning_validation": null,
"validation_distribution": "reasoning only",
"serialized_validation": {
"target_rel": 0.17944146946888725,
"reference_rel": 0.17944146946888725
},
"development_audit": {
"target_rel": 0.14393935014878248,
"reference_rel": 0.14393935014878248,
"windows": [
{
"first_row": 2048,
"rows": 1024,
"reference_sse": 34725.62890625,
"reference_energy": 1756561.625
},
{
"first_row": 3072,
"rows": 1024,
"reference_sse": 39691.34375,
"reference_energy": 1868364.125
},
{
"first_row": 6144,
"rows": 1024,
"reference_sse": 31834.30859375,
"reference_energy": 1786917.0
},
{
"first_row": 7168,
"rows": 1024,
"reference_sse": 37571.3671875,
"reference_energy": 4828909.0
},
{
"first_row": 10240,
"rows": 1024,
"reference_sse": 46532.015625,
"reference_energy": 3304872.5
},
{
"first_row": 11264,
"rows": 1024,
"reference_sse": 30461.689453125,
"reference_energy": 1828629.25
},
{
"first_row": 14336,
"rows": 1024,
"reference_sse": 123385.484375,
"reference_energy": 3342776.25
},
{
"first_row": 15360,
"rows": 1024,
"reference_sse": 26178.98828125,
"reference_energy": 1855729.0
},
{
"first_row": 18432,
"rows": 1024,
"reference_sse": 36549.65625,
"reference_energy": 1693362.5
},
{
"first_row": 19456,
"rows": 1024,
"reference_sse": 37004.50390625,
"reference_energy": 1812719.625
},
{
"first_row": 22528,
"rows": 1024,
"reference_sse": 33518.6796875,
"reference_energy": 1753350.0
},
{
"first_row": 23552,
"rows": 1024,
"reference_sse": 128569.9140625,
"reference_energy": 1764293.5
},
{
"first_row": 26624,
"rows": 1024,
"reference_sse": 40019.1328125,
"reference_energy": 1780996.625
},
{
"first_row": 27648,
"rows": 1024,
"reference_sse": 38410.6484375,
"reference_energy": 3308889.75
},
{
"first_row": 30720,
"rows": 1024,
"reference_sse": 30796.75390625,
"reference_energy": 1715516.5
},
{
"first_row": 31744,
"rows": 1024,
"reference_sse": 34154.2421875,
"reference_energy": 1768831.5
}
]
},
"initial_development_audit": {
"target_rel": 0.1440887977384466,
"reference_rel": 0.1440887977384466,
"windows": [
{
"first_row": 2048,
"rows": 1024,
"reference_sse": 34764.921875,
"reference_energy": 1756561.625
},
{
"first_row": 3072,
"rows": 1024,
"reference_sse": 39737.96875,
"reference_energy": 1868364.125
},
{
"first_row": 6144,
"rows": 1024,
"reference_sse": 31885.28515625,
"reference_energy": 1786917.0
},
{
"first_row": 7168,
"rows": 1024,
"reference_sse": 37725.32421875,
"reference_energy": 4828909.0
},
{
"first_row": 10240,
"rows": 1024,
"reference_sse": 46675.953125,
"reference_energy": 3304872.5
},
{
"first_row": 11264,
"rows": 1024,
"reference_sse": 30556.05859375,
"reference_energy": 1828629.25
},
{
"first_row": 14336,
"rows": 1024,
"reference_sse": 123509.296875,
"reference_energy": 3342776.25
},
{
"first_row": 15360,
"rows": 1024,
"reference_sse": 26212.056640625,
"reference_energy": 1855729.0
},
{
"first_row": 18432,
"rows": 1024,
"reference_sse": 36613.296875,
"reference_energy": 1693362.5
},
{
"first_row": 19456,
"rows": 1024,
"reference_sse": 37102.2734375,
"reference_energy": 1812719.625
},
{
"first_row": 22528,
"rows": 1024,
"reference_sse": 33606.11328125,
"reference_energy": 1753350.0
},
{
"first_row": 23552,
"rows": 1024,
"reference_sse": 128856.453125,
"reference_energy": 1764293.5
},
{
"first_row": 26624,
"rows": 1024,
"reference_sse": 40075.11328125,
"reference_energy": 1780996.625
},
{
"first_row": 27648,
"rows": 1024,
"reference_sse": 38540.6328125,
"reference_energy": 3308889.75
},
{
"first_row": 30720,
"rows": 1024,
"reference_sse": 30864.21484375,
"reference_energy": 1715516.5
},
{
"first_row": 31744,
"rows": 1024,
"reference_sse": 34236.3671875,
"reference_energy": 1768831.5
}
]
},
"best_step": 40,
"steps_run": 60,
"history": [
{
"step": 5,
"validation": {
"target_rel": 0.17955213877982507,
"reference_rel": 0.17955213877982507
},
"reasoning_validation": null,
"training_probe_rel": 0.956788841587352,
"batch_loss": 0.8882136940956116,
"best_step": 5,
"overfit_checks": 0,
"seconds": 49.54285216331482,
"adaptive_schedule": {
"best": 0.17955213877982507,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 5,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 0,
"codebook_values": 1376256,
"scales_changed": 8963230,
"scale_values": 49545216
}
},
{
"step": 10,
"validation": {
"target_rel": 0.17954428603424238,
"reference_rel": 0.17954428603424238
},
"reasoning_validation": null,
"training_probe_rel": 0.956639852673752,
"batch_loss": 0.8859453648328781,
"best_step": 10,
"overfit_checks": 0,
"seconds": 97.16759872436523,
"adaptive_schedule": {
"best": 0.17954428603424238,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 10,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 19117,
"codebook_values": 1376256,
"scales_changed": 17773746,
"scale_values": 49545216
}
},
{
"step": 15,
"validation": {
"target_rel": 0.1795074687981107,
"reference_rel": 0.1795074687981107
},
"reasoning_validation": null,
"training_probe_rel": 0.9553310392604821,
"batch_loss": 0.9645052552223206,
"best_step": 15,
"overfit_checks": 0,
"seconds": 147.1776716709137,
"adaptive_schedule": {
"best": 0.1795074687981107,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 15,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 4280,
"codebook_values": 1376256,
"scales_changed": 22819737,
"scale_values": 49545216
}
},
{
"step": 20,
"validation": {
"target_rel": 0.17946218703314962,
"reference_rel": 0.17946218703314962
},
"reasoning_validation": null,
"training_probe_rel": 0.9549531854377901,
"batch_loss": 0.9224357903003693,
"best_step": 20,
"overfit_checks": 0,
"seconds": 206.17729806900024,
"adaptive_schedule": {
"best": 0.17946218703314962,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 20,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 4046,
"codebook_values": 1376256,
"scales_changed": 26117143,
"scale_values": 49545216
}
},
{
"step": 25,
"validation": {
"target_rel": 0.17944604789492144,
"reference_rel": 0.17944604789492144
},
"reasoning_validation": null,
"training_probe_rel": 0.9546469707717397,
"batch_loss": 0.9236540347337723,
"best_step": 25,
"overfit_checks": 0,
"seconds": 256.5486056804657,
"adaptive_schedule": {
"best": 0.17944604789492144,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 25,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 3774,
"codebook_values": 1376256,
"scales_changed": 28405312,
"scale_values": 49545216
}
},
{
"step": 30,
"validation": {
"target_rel": 0.17944675736395735,
"reference_rel": 0.17944675736395735
},
"reasoning_validation": null,
"training_probe_rel": 0.9544466925031139,
"batch_loss": 0.9143165946006775,
"best_step": 25,
"overfit_checks": 0,
"seconds": 304.2834966182709,
"adaptive_schedule": {
"best": 0.17944604789492144,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 25,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 3477,
"codebook_values": 1376256,
"scales_changed": 30064479,
"scale_values": 49545216
}
},
{
"step": 35,
"validation": {
"target_rel": 0.1794431062598024,
"reference_rel": 0.1794431062598024
},
"reasoning_validation": null,
"training_probe_rel": 0.9542527701485309,
"batch_loss": 0.9843097925186157,
"best_step": 35,
"overfit_checks": 0,
"seconds": 356.05538535118103,
"adaptive_schedule": {
"best": 0.1794431062598024,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 35,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 3073,
"codebook_values": 1376256,
"scales_changed": 31322298,
"scale_values": 49545216
}
},
{
"step": 40,
"validation": {
"target_rel": 0.17944146946888725,
"reference_rel": 0.17944146946888725
},
"reasoning_validation": null,
"training_probe_rel": 0.9541932609492495,
"batch_loss": 0.8654253333806992,
"best_step": 40,
"overfit_checks": 0,
"seconds": 417.2343602180481,
"adaptive_schedule": {
"best": 0.17944146946888725,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 40,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 3131,
"codebook_values": 1376256,
"scales_changed": 32309979,
"scale_values": 49545216
}
},
{
"step": 45,
"validation": {
"target_rel": 0.1794600285159272,
"reference_rel": 0.1794600285159272
},
"reasoning_validation": null,
"training_probe_rel": 0.9540594464938058,
"batch_loss": 0.8673814982175827,
"best_step": 40,
"overfit_checks": 0,
"seconds": 471.17846393585205,
"adaptive_schedule": {
"best": 0.17944146946888725,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 40,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 2876,
"codebook_values": 1376256,
"scales_changed": 33099761,
"scale_values": 49545216
}
},
{
"step": 50,
"validation": {
"target_rel": 0.17945150701507015,
"reference_rel": 0.17945150701507015
},
"reasoning_validation": null,
"training_probe_rel": 0.9538829247663951,
"batch_loss": 0.9367623180150986,
"best_step": 40,
"overfit_checks": 0,
"seconds": 517.8566815853119,
"adaptive_schedule": {
"best": 0.17944146946888725,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 40,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 1150,
"codebook_values": 1376256,
"scales_changed": 33254796,
"scale_values": 49545216
}
},
{
"step": 55,
"validation": {
"target_rel": 0.17945296576529257,
"reference_rel": 0.17945296576529257
},
"reasoning_validation": null,
"training_probe_rel": 0.953899362172705,
"batch_loss": 1.0069809556007385,
"best_step": 40,
"overfit_checks": 0,
"seconds": 565.3292081356049,
"adaptive_schedule": {
"best": 0.17944146946888725,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 40,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 1716,
"codebook_values": 1376256,
"scales_changed": 33384002,
"scale_values": 49545216
}
},
{
"step": 60,
"validation": {
"target_rel": 0.17945947319096286,
"reference_rel": 0.17945947319096286
},
"reasoning_validation": null,
"training_probe_rel": 0.953901662182793,
"batch_loss": 0.9198732972145081,
"best_step": 40,
"overfit_checks": 0,
"seconds": 628.189727306366,
"adaptive_schedule": {
"best": 0.17944146946888725,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 40,
"early_drop": false
},
"adaptive_stop": true,
"quantized_movement": {
"codebook_changed": 1931,
"codebook_values": 1376256,
"scales_changed": 33495314,
"scale_values": 49545216
}
}
],
"index_reassignments": [
{
"method": "expert_parallel_output_gradient_prefix_backtracking_v1",
"proposals": 320,
"accepted": 267,
"changed_groups": 201029,
"training_before": 553750.5,
"training_after": 553249.9375,
"acceptance": "proposal and separate training check batches",
"step": 20
},
{
"method": "expert_parallel_output_gradient_prefix_backtracking_v1",
"proposals": 326,
"accepted": 278,
"changed_groups": 187929,
"training_before": 463282.875,
"training_after": 462889.8125,
"acceptance": "proposal and separate training check batches",
"step": 40
},
{
"method": "expert_parallel_output_gradient_prefix_backtracking_v1",
"proposals": 322,
"accepted": 270,
"changed_groups": 211275,
"training_before": 487197.3125,
"training_after": 486880.46875,
"acceptance": "proposal and separate training check batches",
"step": 60
}
],
"coverage_threshold": 256,
"checkpoint_selection": "held-out original reference trajectory for both arms",
"world_size": 8,
"microbatch": 65536,
"proposal_communication": "sparse routed rows",
"parallelism": "expert-sharded coupled output",
"arithmetic": "fp4_planes4_fp16_boundaries_v1",
"scope": "text-only, cached prompt partitions; historical audit is development only",
"train_rows": 18001846,
"validation_rows": 16384,
"audit_rows": 16384,
"complete": true
}