jarrelscy's picture
Publish GLM-5.3 NVFP4 / ARVQ v2 same_input fp16-block hybrid: non-cold structure seed (part 2)
868041e verified
Raw History Blame
22.5 kB
{
"layer": 51,
"target": "reference",
"recipe": "full_corpus_sequence_pass_v1",
"block_scale_dtype": "fp8_e4m3",
"batch_tokens": 262144,
"sequence_order_sha256": "edebb22a5f7b34a6cbfa59c58aab9f111651d6ce0926212b5237dea3ef0ba485",
"training_sequence_count": 17580,
"training_passes": 0.8737237281109949,
"training_tokens_seen": 15728640,
"adaptive_schedule": {
"best": 0.29780141555672285,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 2,
"last_best": 20,
"early_drop": false
},
"refinement": null,
"lr_decay_after": 45,
"lr_decay_factor": 0.25,
"export_only": false,
"training_output_export": "fixed validation/audit outputs only; full training trajectory export not performed",
"codebook_lr": 0.048,
"scale_lr": 0.032,
"quantized_movement": {
"codebook_changed": 4426,
"codebook_values": 1531904,
"scales_changed": 25077522,
"scale_values": 55148544
},
"initial_validation": {
"target_rel": 0.2979071394740392,
"reference_rel": 0.2979071394740392,
"windows": [
{
"first_row": 0,
"rows": 1024,
"reference_sse": 332088.25,
"reference_energy": 3862912.0
},
{
"first_row": 1024,
"rows": 1024,
"reference_sse": 374147.09375,
"reference_energy": 3797315.0
},
{
"first_row": 4096,
"rows": 1024,
"reference_sse": 528405.625,
"reference_energy": 4970908.5
},
{
"first_row": 5120,
"rows": 1024,
"reference_sse": 384278.625,
"reference_energy": 3635619.5
},
{
"first_row": 8192,
"rows": 1024,
"reference_sse": 304144.6875,
"reference_energy": 3868136.0
},
{
"first_row": 9216,
"rows": 1024,
"reference_sse": 331265.75,
"reference_energy": 6895785.5
},
{
"first_row": 12288,
"rows": 1024,
"reference_sse": 395230.5625,
"reference_energy": 3764681.5
},
{
"first_row": 13312,
"rows": 1024,
"reference_sse": 353063.40625,
"reference_energy": 3723862.5
},
{
"first_row": 16384,
"rows": 1024,
"reference_sse": 315180.875,
"reference_energy": 3641669.25
},
{
"first_row": 17408,
"rows": 1024,
"reference_sse": 314756.84375,
"reference_energy": 3774562.75
},
{
"first_row": 20480,
"rows": 1024,
"reference_sse": 322242.875,
"reference_energy": 3866016.0
},
{
"first_row": 21504,
"rows": 1024,
"reference_sse": 440806.125,
"reference_energy": 5090171.0
},
{
"first_row": 24576,
"rows": 1024,
"reference_sse": 435245.53125,
"reference_energy": 3797902.25
},
{
"first_row": 25600,
"rows": 1024,
"reference_sse": 272304.1875,
"reference_energy": 3757266.75
},
{
"first_row": 28672,
"rows": 1024,
"reference_sse": 261563.6875,
"reference_energy": 3623433.0
},
{
"first_row": 29696,
"rows": 1024,
"reference_sse": 473923.375,
"reference_energy": 3718326.5
}
]
},
"final_validation": {
"target_rel": 0.29780141555672285,
"reference_rel": 0.29780141555672285,
"windows": [
{
"first_row": 0,
"rows": 1024,
"reference_sse": 331801.40625,
"reference_energy": 3862912.0
},
{
"first_row": 1024,
"rows": 1024,
"reference_sse": 373735.46875,
"reference_energy": 3797315.0
},
{
"first_row": 4096,
"rows": 1024,
"reference_sse": 529257.9375,
"reference_energy": 4970908.5
},
{
"first_row": 5120,
"rows": 1024,
"reference_sse": 383964.75,
"reference_energy": 3635619.5
},
{
"first_row": 8192,
"rows": 1024,
"reference_sse": 304770.875,
"reference_energy": 3868136.0
},
{
"first_row": 9216,
"rows": 1024,
"reference_sse": 330452.15625,
"reference_energy": 6895785.5
},
{
"first_row": 12288,
"rows": 1024,
"reference_sse": 394611.09375,
"reference_energy": 3764681.5
},
{
"first_row": 13312,
"rows": 1024,
"reference_sse": 352550.46875,
"reference_energy": 3723862.5
},
{
"first_row": 16384,
"rows": 1024,
"reference_sse": 314761.90625,
"reference_energy": 3641669.25
},
{
"first_row": 17408,
"rows": 1024,
"reference_sse": 314292.75,
"reference_energy": 3774562.75
},
{
"first_row": 20480,
"rows": 1024,
"reference_sse": 321836.3125,
"reference_energy": 3866016.0
},
{
"first_row": 21504,
"rows": 1024,
"reference_sse": 440142.6875,
"reference_energy": 5090171.0
},
{
"first_row": 24576,
"rows": 1024,
"reference_sse": 434727.625,
"reference_energy": 3797902.25
},
{
"first_row": 25600,
"rows": 1024,
"reference_sse": 271897.875,
"reference_energy": 3757266.75
},
{
"first_row": 28672,
"rows": 1024,
"reference_sse": 261096.59375,
"reference_energy": 3623433.0
},
{
"first_row": 29696,
"rows": 1024,
"reference_sse": 474604.1875,
"reference_energy": 3718326.5
}
]
},
"initial_reasoning_validation": null,
"final_reasoning_validation": null,
"validation_distribution": "reasoning only",
"serialized_validation": {
"target_rel": 0.29780141555672285,
"reference_rel": 0.29780141555672285
},
"development_audit": {
"target_rel": 0.289689428534497,
"reference_rel": 0.289689428534497,
"windows": [
{
"first_row": 2048,
"rows": 1024,
"reference_sse": 324628.25,
"reference_energy": 3901042.75
},
{
"first_row": 3072,
"rows": 1024,
"reference_sse": 399303.1875,
"reference_energy": 3673259.75
},
{
"first_row": 6144,
"rows": 1024,
"reference_sse": 281599.21875,
"reference_energy": 3908435.75
},
{
"first_row": 7168,
"rows": 1024,
"reference_sse": 328679.8125,
"reference_energy": 6845727.0
},
{
"first_row": 10240,
"rows": 1024,
"reference_sse": 397984.0,
"reference_energy": 4925583.0
},
{
"first_row": 11264,
"rows": 1024,
"reference_sse": 293913.125,
"reference_energy": 3936069.5
},
{
"first_row": 14336,
"rows": 1024,
"reference_sse": 458189.5,
"reference_energy": 5167311.5
},
{
"first_row": 15360,
"rows": 1024,
"reference_sse": 287973.15625,
"reference_energy": 3744658.0
},
{
"first_row": 18432,
"rows": 1024,
"reference_sse": 366283.25,
"reference_energy": 3801818.25
},
{
"first_row": 19456,
"rows": 1024,
"reference_sse": 380389.0,
"reference_energy": 3705748.75
},
{
"first_row": 22528,
"rows": 1024,
"reference_sse": 367526.375,
"reference_energy": 3644467.5
},
{
"first_row": 23552,
"rows": 1024,
"reference_sse": 349091.0625,
"reference_energy": 3801917.5
},
{
"first_row": 26624,
"rows": 1024,
"reference_sse": 398096.59375,
"reference_energy": 3682957.0
},
{
"first_row": 27648,
"rows": 1024,
"reference_sse": 360063.28125,
"reference_energy": 5071052.0
},
{
"first_row": 30720,
"rows": 1024,
"reference_sse": 310555.3125,
"reference_energy": 3674003.0
},
{
"first_row": 31744,
"rows": 1024,
"reference_sse": 338124.96875,
"reference_energy": 3751440.25
}
]
},
"initial_development_audit": {
"target_rel": 0.28985552770850564,
"reference_rel": 0.28985552770850564,
"windows": [
{
"first_row": 2048,
"rows": 1024,
"reference_sse": 325079.96875,
"reference_energy": 3901042.75
},
{
"first_row": 3072,
"rows": 1024,
"reference_sse": 399865.75,
"reference_energy": 3673259.75
},
{
"first_row": 6144,
"rows": 1024,
"reference_sse": 281890.5625,
"reference_energy": 3908435.75
},
{
"first_row": 7168,
"rows": 1024,
"reference_sse": 329508.78125,
"reference_energy": 6845727.0
},
{
"first_row": 10240,
"rows": 1024,
"reference_sse": 398666.5,
"reference_energy": 4925583.0
},
{
"first_row": 11264,
"rows": 1024,
"reference_sse": 294380.5,
"reference_energy": 3936069.5
},
{
"first_row": 14336,
"rows": 1024,
"reference_sse": 458181.5625,
"reference_energy": 5167311.5
},
{
"first_row": 15360,
"rows": 1024,
"reference_sse": 288576.0,
"reference_energy": 3744658.0
},
{
"first_row": 18432,
"rows": 1024,
"reference_sse": 366661.59375,
"reference_energy": 3801818.25
},
{
"first_row": 19456,
"rows": 1024,
"reference_sse": 380875.375,
"reference_energy": 3705748.75
},
{
"first_row": 22528,
"rows": 1024,
"reference_sse": 367832.46875,
"reference_energy": 3644467.5
},
{
"first_row": 23552,
"rows": 1024,
"reference_sse": 348696.4375,
"reference_energy": 3801917.5
},
{
"first_row": 26624,
"rows": 1024,
"reference_sse": 398536.3125,
"reference_energy": 3682957.0
},
{
"first_row": 27648,
"rows": 1024,
"reference_sse": 360562.5625,
"reference_energy": 5071052.0
},
{
"first_row": 30720,
"rows": 1024,
"reference_sse": 310858.3125,
"reference_energy": 3674003.0
},
{
"first_row": 31744,
"rows": 1024,
"reference_sse": 338699.625,
"reference_energy": 3751440.25
}
]
},
"best_step": 20,
"steps_run": 60,
"history": [
{
"step": 5,
"validation": {
"target_rel": 0.29786036376710184,
"reference_rel": 0.29786036376710184
},
"reasoning_validation": null,
"training_probe_rel": 0.9867873316130592,
"batch_loss": 0.9468852132558823,
"best_step": 5,
"overfit_checks": 0,
"seconds": 55.20050001144409,
"adaptive_schedule": {
"best": 0.29786036376710184,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 5,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 0,
"codebook_values": 1531904,
"scales_changed": 6626522,
"scale_values": 55148544
}
},
{
"step": 10,
"validation": {
"target_rel": 0.29782093347786454,
"reference_rel": 0.29782093347786454
},
"reasoning_validation": null,
"training_probe_rel": 0.9863402257730945,
"batch_loss": 0.9380551129579544,
"best_step": 10,
"overfit_checks": 0,
"seconds": 108.53682112693787,
"adaptive_schedule": {
"best": 0.29782093347786454,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 10,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 32233,
"codebook_values": 1531904,
"scales_changed": 15147891,
"scale_values": 55148544
}
},
{
"step": 15,
"validation": {
"target_rel": 0.2978162483045699,
"reference_rel": 0.2978162483045699
},
"reasoning_validation": null,
"training_probe_rel": 0.9856866092800709,
"batch_loss": 0.9560574591159821,
"best_step": 15,
"overfit_checks": 0,
"seconds": 162.51102685928345,
"adaptive_schedule": {
"best": 0.2978162483045699,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 15,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 5306,
"codebook_values": 1531904,
"scales_changed": 20932783,
"scale_values": 55148544
}
},
{
"step": 20,
"validation": {
"target_rel": 0.29780141555672285,
"reference_rel": 0.29780141555672285
},
"reasoning_validation": null,
"training_probe_rel": 0.9853117901589347,
"batch_loss": 0.9602041244506836,
"best_step": 20,
"overfit_checks": 0,
"seconds": 236.80384755134583,
"adaptive_schedule": {
"best": 0.29780141555672285,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 20,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 4426,
"codebook_values": 1531904,
"scales_changed": 25077522,
"scale_values": 55148544
}
},
{
"step": 25,
"validation": {
"target_rel": 0.29786955763617345,
"reference_rel": 0.29786955763617345
},
"reasoning_validation": null,
"training_probe_rel": 0.9849560468410179,
"batch_loss": 0.9678227454423904,
"best_step": 20,
"overfit_checks": 0,
"seconds": 295.48638892173767,
"adaptive_schedule": {
"best": 0.29780141555672285,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 20,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 6588,
"codebook_values": 1531904,
"scales_changed": 28125508,
"scale_values": 55148544
}
},
{
"step": 30,
"validation": {
"target_rel": 0.2979763453569969,
"reference_rel": 0.2979763453569969
},
"reasoning_validation": null,
"training_probe_rel": 0.9846206497064001,
"batch_loss": 0.9535466432571411,
"best_step": 20,
"overfit_checks": 0,
"seconds": 349.1486179828644,
"adaptive_schedule": {
"best": 0.29780141555672285,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 20,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 9983,
"codebook_values": 1531904,
"scales_changed": 30441454,
"scale_values": 55148544
}
},
{
"step": 35,
"validation": {
"target_rel": 0.29797466994455596,
"reference_rel": 0.29797466994455596
},
"reasoning_validation": null,
"training_probe_rel": 0.9845241691208498,
"batch_loss": 0.9720050096511841,
"best_step": 20,
"overfit_checks": 0,
"seconds": 404.4410881996155,
"adaptive_schedule": {
"best": 0.29780141555672285,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 20,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 11474,
"codebook_values": 1531904,
"scales_changed": 32249871,
"scale_values": 55148544
}
},
{
"step": 40,
"validation": {
"target_rel": 0.2981529600586089,
"reference_rel": 0.2981529600586089
},
"reasoning_validation": null,
"training_probe_rel": 0.9843603312778929,
"batch_loss": 0.9430856853723526,
"best_step": 20,
"overfit_checks": 0,
"seconds": 474.8598828315735,
"adaptive_schedule": {
"best": 0.29780141555672285,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 1,
"last_best": 20,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 13190,
"codebook_values": 1531904,
"scales_changed": 33702606,
"scale_values": 55148544
}
},
{
"step": 45,
"validation": {
"target_rel": 0.2983179872791641,
"reference_rel": 0.2983179872791641
},
"reasoning_validation": null,
"training_probe_rel": 0.984146157822943,
"batch_loss": 0.9436440169811249,
"best_step": 20,
"overfit_checks": 0,
"seconds": 534.4382810592651,
"adaptive_schedule": {
"best": 0.29780141555672285,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 2,
"last_best": 20,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 12877,
"codebook_values": 1531904,
"scales_changed": 34880297,
"scale_values": 55148544
}
},
{
"step": 50,
"validation": {
"target_rel": 0.29821026185335286,
"reference_rel": 0.29821026185335286
},
"reasoning_validation": null,
"training_probe_rel": 0.9841426860946676,
"batch_loss": 0.9575401097536087,
"best_step": 20,
"overfit_checks": 0,
"seconds": 588.8291883468628,
"adaptive_schedule": {
"best": 0.29780141555672285,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 2,
"last_best": 20,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 10219,
"codebook_values": 1531904,
"scales_changed": 35119904,
"scale_values": 55148544
}
},
{
"step": 55,
"validation": {
"target_rel": 0.29823862651414923,
"reference_rel": 0.29823862651414923
},
"reasoning_validation": null,
"training_probe_rel": 0.9841351659978934,
"batch_loss": 0.9788200855255127,
"best_step": 20,
"overfit_checks": 0,
"seconds": 645.0570511817932,
"adaptive_schedule": {
"best": 0.29780141555672285,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 2,
"last_best": 20,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 10635,
"codebook_values": 1531904,
"scales_changed": 35322218,
"scale_values": 55148544
}
},
{
"step": 60,
"validation": {
"target_rel": 0.2983731320842526,
"reference_rel": 0.2983731320842526
},
"reasoning_validation": null,
"training_probe_rel": 0.984080575291871,
"batch_loss": 0.9483949840068817,
"best_step": 20,
"overfit_checks": 0,
"seconds": 720.636358499527,
"adaptive_schedule": {
"best": 0.29780141555672285,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 2,
"last_best": 20,
"early_drop": false
},
"adaptive_stop": true,
"quantized_movement": {
"codebook_changed": 12314,
"codebook_values": 1531904,
"scales_changed": 35500486,
"scale_values": 55148544
}
}
],
"index_reassignments": [
{
"method": "expert_parallel_output_gradient_prefix_backtracking_v1",
"proposals": 349,
"accepted": 300,
"changed_groups": 208910,
"training_before": 3196874.25,
"training_after": 3193211.0,
"acceptance": "proposal and separate training check batches",
"step": 20
},
{
"method": "expert_parallel_output_gradient_prefix_backtracking_v1",
"proposals": 348,
"accepted": 307,
"changed_groups": 220062,
"training_before": 3170806.0,
"training_after": 3167004.5,
"acceptance": "proposal and separate training check batches",
"step": 40
},
{
"method": "expert_parallel_output_gradient_prefix_backtracking_v1",
"proposals": 347,
"accepted": 301,
"changed_groups": 197469,
"training_before": 3129976.5,
"training_after": 3126557.25,
"acceptance": "proposal and separate training check batches",
"step": 60
}
],
"coverage_threshold": 256,
"checkpoint_selection": "held-out original reference trajectory for both arms",
"world_size": 8,
"microbatch": 65536,
"proposal_communication": "sparse routed rows",
"parallelism": "expert-sharded coupled output",
"arithmetic": "fp4_planes4_fp16_boundaries_v1",
"scope": "text-only, cached prompt partitions; historical audit is development only",
"train_rows": 18001846,
"validation_rows": 16384,
"audit_rows": 16384,
"complete": true
}