jarrelscy's picture
Publish GLM-5.3 NVFP4 / ARVQ v2 same_input fp16-block hybrid: non-cold structure seed (part 2)
868041e verified
Raw History Blame
22.5 kB
{
"layer": 49,
"target": "reference",
"recipe": "full_corpus_sequence_pass_v1",
"block_scale_dtype": "fp8_e4m3",
"batch_tokens": 262144,
"sequence_order_sha256": "edebb22a5f7b34a6cbfa59c58aab9f111651d6ce0926212b5237dea3ef0ba485",
"training_sequence_count": 17580,
"training_passes": 0.8737237281109949,
"training_tokens_seen": 15728640,
"adaptive_schedule": {
"best": 0.2789341851468615,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 15,
"early_drop": false
},
"refinement": null,
"lr_decay_after": 45,
"lr_decay_factor": 0.25,
"export_only": false,
"training_output_export": "fixed validation/audit outputs only; full training trajectory export not performed",
"codebook_lr": 0.048,
"scale_lr": 0.032,
"quantized_movement": {
"codebook_changed": 7831,
"codebook_values": 1384448,
"scales_changed": 19553214,
"scale_values": 49840128
},
"initial_validation": {
"target_rel": 0.2790533744336196,
"reference_rel": 0.2790533744336196,
"windows": [
{
"first_row": 0,
"rows": 1024,
"reference_sse": 239546.734375,
"reference_energy": 3127584.0
},
{
"first_row": 1024,
"rows": 1024,
"reference_sse": 271579.0625,
"reference_energy": 3055043.5
},
{
"first_row": 4096,
"rows": 1024,
"reference_sse": 414678.28125,
"reference_energy": 4414620.0
},
{
"first_row": 5120,
"rows": 1024,
"reference_sse": 275810.1875,
"reference_energy": 2991313.25
},
{
"first_row": 8192,
"rows": 1024,
"reference_sse": 246048.265625,
"reference_energy": 3113381.75
},
{
"first_row": 9216,
"rows": 1024,
"reference_sse": 250375.296875,
"reference_energy": 6238750.0
},
{
"first_row": 12288,
"rows": 1024,
"reference_sse": 282899.375,
"reference_energy": 3041122.5
},
{
"first_row": 13312,
"rows": 1024,
"reference_sse": 247937.015625,
"reference_energy": 3074767.25
},
{
"first_row": 16384,
"rows": 1024,
"reference_sse": 229615.78125,
"reference_energy": 3055161.0
},
{
"first_row": 17408,
"rows": 1024,
"reference_sse": 225775.40625,
"reference_energy": 3111365.0
},
{
"first_row": 20480,
"rows": 1024,
"reference_sse": 228839.28125,
"reference_energy": 3148168.0
},
{
"first_row": 21504,
"rows": 1024,
"reference_sse": 327636.0625,
"reference_energy": 4539791.5
},
{
"first_row": 24576,
"rows": 1024,
"reference_sse": 301998.25,
"reference_energy": 3120925.75
},
{
"first_row": 25600,
"rows": 1024,
"reference_sse": 195641.140625,
"reference_energy": 3101186.0
},
{
"first_row": 28672,
"rows": 1024,
"reference_sse": 185797.15625,
"reference_energy": 2980401.0
},
{
"first_row": 29696,
"rows": 1024,
"reference_sse": 375261.75,
"reference_energy": 3098897.5
}
]
},
"final_validation": {
"target_rel": 0.2789341851468615,
"reference_rel": 0.2789341851468615,
"windows": [
{
"first_row": 0,
"rows": 1024,
"reference_sse": 239376.9375,
"reference_energy": 3127584.0
},
{
"first_row": 1024,
"rows": 1024,
"reference_sse": 271369.40625,
"reference_energy": 3055043.5
},
{
"first_row": 4096,
"rows": 1024,
"reference_sse": 414252.5625,
"reference_energy": 4414620.0
},
{
"first_row": 5120,
"rows": 1024,
"reference_sse": 275483.75,
"reference_energy": 2991313.25
},
{
"first_row": 8192,
"rows": 1024,
"reference_sse": 245982.5625,
"reference_energy": 3113381.75
},
{
"first_row": 9216,
"rows": 1024,
"reference_sse": 250045.71875,
"reference_energy": 6238750.0
},
{
"first_row": 12288,
"rows": 1024,
"reference_sse": 282675.625,
"reference_energy": 3041122.5
},
{
"first_row": 13312,
"rows": 1024,
"reference_sse": 247694.296875,
"reference_energy": 3074767.25
},
{
"first_row": 16384,
"rows": 1024,
"reference_sse": 229362.234375,
"reference_energy": 3055161.0
},
{
"first_row": 17408,
"rows": 1024,
"reference_sse": 225588.28125,
"reference_energy": 3111365.0
},
{
"first_row": 20480,
"rows": 1024,
"reference_sse": 228648.59375,
"reference_energy": 3148168.0
},
{
"first_row": 21504,
"rows": 1024,
"reference_sse": 327356.625,
"reference_energy": 4539791.5
},
{
"first_row": 24576,
"rows": 1024,
"reference_sse": 301652.4375,
"reference_energy": 3120925.75
},
{
"first_row": 25600,
"rows": 1024,
"reference_sse": 195523.171875,
"reference_energy": 3101186.0
},
{
"first_row": 28672,
"rows": 1024,
"reference_sse": 185731.375,
"reference_energy": 2980401.0
},
{
"first_row": 29696,
"rows": 1024,
"reference_sse": 375023.5,
"reference_energy": 3098897.5
}
]
},
"initial_reasoning_validation": null,
"final_reasoning_validation": null,
"validation_distribution": "reasoning only",
"serialized_validation": {
"target_rel": 0.2789341851468615,
"reference_rel": 0.2789341851468615
},
"development_audit": {
"target_rel": 0.26909877518246317,
"reference_rel": 0.26909877518246317,
"windows": [
{
"first_row": 2048,
"rows": 1024,
"reference_sse": 233430.84375,
"reference_energy": 3102338.5
},
{
"first_row": 3072,
"rows": 1024,
"reference_sse": 291834.46875,
"reference_energy": 3048998.0
},
{
"first_row": 6144,
"rows": 1024,
"reference_sse": 203986.65625,
"reference_energy": 3101616.0
},
{
"first_row": 7168,
"rows": 1024,
"reference_sse": 241080.0625,
"reference_energy": 6165425.5
},
{
"first_row": 10240,
"rows": 1024,
"reference_sse": 292317.71875,
"reference_energy": 4413415.0
},
{
"first_row": 11264,
"rows": 1024,
"reference_sse": 216040.53125,
"reference_energy": 3204521.0
},
{
"first_row": 14336,
"rows": 1024,
"reference_sse": 348673.09375,
"reference_energy": 4569804.0
},
{
"first_row": 15360,
"rows": 1024,
"reference_sse": 205568.6875,
"reference_energy": 3112773.0
},
{
"first_row": 18432,
"rows": 1024,
"reference_sse": 263765.625,
"reference_energy": 3135705.75
},
{
"first_row": 19456,
"rows": 1024,
"reference_sse": 273368.4375,
"reference_energy": 3091696.25
},
{
"first_row": 22528,
"rows": 1024,
"reference_sse": 256843.0625,
"reference_energy": 3021575.0
},
{
"first_row": 23552,
"rows": 1024,
"reference_sse": 271274.96875,
"reference_energy": 3160500.0
},
{
"first_row": 26624,
"rows": 1024,
"reference_sse": 278960.5625,
"reference_energy": 3048948.75
},
{
"first_row": 27648,
"rows": 1024,
"reference_sse": 265827.8125,
"reference_energy": 4502916.0
},
{
"first_row": 30720,
"rows": 1024,
"reference_sse": 220938.796875,
"reference_energy": 3039270.25
},
{
"first_row": 31744,
"rows": 1024,
"reference_sse": 253886.046875,
"reference_energy": 3145037.0
}
]
},
"initial_development_audit": {
"target_rel": 0.26922334258438857,
"reference_rel": 0.26922334258438857,
"windows": [
{
"first_row": 2048,
"rows": 1024,
"reference_sse": 233539.5625,
"reference_energy": 3102338.5
},
{
"first_row": 3072,
"rows": 1024,
"reference_sse": 292047.65625,
"reference_energy": 3048998.0
},
{
"first_row": 6144,
"rows": 1024,
"reference_sse": 204020.671875,
"reference_energy": 3101616.0
},
{
"first_row": 7168,
"rows": 1024,
"reference_sse": 241374.09375,
"reference_energy": 6165425.5
},
{
"first_row": 10240,
"rows": 1024,
"reference_sse": 292741.0625,
"reference_energy": 4413415.0
},
{
"first_row": 11264,
"rows": 1024,
"reference_sse": 216380.296875,
"reference_energy": 3204521.0
},
{
"first_row": 14336,
"rows": 1024,
"reference_sse": 348990.375,
"reference_energy": 4569804.0
},
{
"first_row": 15360,
"rows": 1024,
"reference_sse": 205735.25,
"reference_energy": 3112773.0
},
{
"first_row": 18432,
"rows": 1024,
"reference_sse": 264006.9375,
"reference_energy": 3135705.75
},
{
"first_row": 19456,
"rows": 1024,
"reference_sse": 273751.375,
"reference_energy": 3091696.25
},
{
"first_row": 22528,
"rows": 1024,
"reference_sse": 257065.546875,
"reference_energy": 3021575.0
},
{
"first_row": 23552,
"rows": 1024,
"reference_sse": 271455.90625,
"reference_energy": 3160500.0
},
{
"first_row": 26624,
"rows": 1024,
"reference_sse": 279204.8125,
"reference_energy": 3048948.75
},
{
"first_row": 27648,
"rows": 1024,
"reference_sse": 266055.59375,
"reference_energy": 4502916.0
},
{
"first_row": 30720,
"rows": 1024,
"reference_sse": 221077.265625,
"reference_energy": 3039270.25
},
{
"first_row": 31744,
"rows": 1024,
"reference_sse": 254164.15625,
"reference_energy": 3145037.0
}
]
},
"best_step": 15,
"steps_run": 60,
"history": [
{
"step": 5,
"validation": {
"target_rel": 0.2789787637197307,
"reference_rel": 0.2789787637197307
},
"reasoning_validation": null,
"training_probe_rel": 0.9809910051598877,
"batch_loss": 0.9401408582925797,
"best_step": 5,
"overfit_checks": 0,
"seconds": 52.91561245918274,
"adaptive_schedule": {
"best": 0.2789787637197307,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 5,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 0,
"codebook_values": 1384448,
"scales_changed": 5905920,
"scale_values": 49840128
}
},
{
"step": 10,
"validation": {
"target_rel": 0.278939181340891,
"reference_rel": 0.278939181340891
},
"reasoning_validation": null,
"training_probe_rel": 0.9803890528977595,
"batch_loss": 0.9350131005048752,
"best_step": 10,
"overfit_checks": 0,
"seconds": 103.83308172225952,
"adaptive_schedule": {
"best": 0.278939181340891,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 10,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 19720,
"codebook_values": 1384448,
"scales_changed": 14136486,
"scale_values": 49840128
}
},
{
"step": 15,
"validation": {
"target_rel": 0.2789341851468615,
"reference_rel": 0.2789341851468615
},
"reasoning_validation": null,
"training_probe_rel": 0.9797758257548224,
"batch_loss": 0.9526158571243286,
"best_step": 15,
"overfit_checks": 0,
"seconds": 153.89956426620483,
"adaptive_schedule": {
"best": 0.2789341851468615,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 15,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 7831,
"codebook_values": 1384448,
"scales_changed": 19553214,
"scale_values": 49840128
}
},
{
"step": 20,
"validation": {
"target_rel": 0.27896381909603146,
"reference_rel": 0.27896381909603146
},
"reasoning_validation": null,
"training_probe_rel": 0.9795485299042581,
"batch_loss": 0.9547634869813919,
"best_step": 15,
"overfit_checks": 0,
"seconds": 219.37319946289062,
"adaptive_schedule": {
"best": 0.2789341851468615,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 15,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 9352,
"codebook_values": 1384448,
"scales_changed": 23348194,
"scale_values": 49840128
}
},
{
"step": 25,
"validation": {
"target_rel": 0.27898402082726054,
"reference_rel": 0.27898402082726054
},
"reasoning_validation": null,
"training_probe_rel": 0.9793855161030279,
"batch_loss": 0.9532788842916489,
"best_step": 15,
"overfit_checks": 0,
"seconds": 273.9892089366913,
"adaptive_schedule": {
"best": 0.2789341851468615,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 15,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 9779,
"codebook_values": 1384448,
"scales_changed": 26090339,
"scale_values": 49840128
}
},
{
"step": 30,
"validation": {
"target_rel": 0.2789997880053497,
"reference_rel": 0.2789997880053497
},
"reasoning_validation": null,
"training_probe_rel": 0.9792641539192678,
"batch_loss": 0.9432947486639023,
"best_step": 15,
"overfit_checks": 0,
"seconds": 328.1168246269226,
"adaptive_schedule": {
"best": 0.2789341851468615,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 15,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 10622,
"codebook_values": 1384448,
"scales_changed": 28125764,
"scale_values": 49840128
}
},
{
"step": 35,
"validation": {
"target_rel": 0.2789874240731857,
"reference_rel": 0.2789874240731857
},
"reasoning_validation": null,
"training_probe_rel": 0.9790952198308966,
"batch_loss": 0.9636296480894089,
"best_step": 15,
"overfit_checks": 0,
"seconds": 383.829313993454,
"adaptive_schedule": {
"best": 0.2789341851468615,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 15,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 11855,
"codebook_values": 1384448,
"scales_changed": 29691548,
"scale_values": 49840128
}
},
{
"step": 40,
"validation": {
"target_rel": 0.27898437941265525,
"reference_rel": 0.27898437941265525
},
"reasoning_validation": null,
"training_probe_rel": 0.9789936916694248,
"batch_loss": 0.934404045343399,
"best_step": 15,
"overfit_checks": 0,
"seconds": 453.98327803611755,
"adaptive_schedule": {
"best": 0.2789341851468615,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 15,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 12827,
"codebook_values": 1384448,
"scales_changed": 30924895,
"scale_values": 49840128
}
},
{
"step": 45,
"validation": {
"target_rel": 0.2790096212461687,
"reference_rel": 0.2790096212461687
},
"reasoning_validation": null,
"training_probe_rel": 0.978893544654874,
"batch_loss": 0.9342323243618011,
"best_step": 15,
"overfit_checks": 0,
"seconds": 511.8981418609619,
"adaptive_schedule": {
"best": 0.2789341851468615,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 15,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 12598,
"codebook_values": 1384448,
"scales_changed": 31919593,
"scale_values": 49840128
}
},
{
"step": 50,
"validation": {
"target_rel": 0.2789836820219663,
"reference_rel": 0.2789836820219663
},
"reasoning_validation": null,
"training_probe_rel": 0.9787584699268429,
"batch_loss": 0.9489616900682449,
"best_step": 15,
"overfit_checks": 0,
"seconds": 565.9121353626251,
"adaptive_schedule": {
"best": 0.2789341851468615,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 15,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 11151,
"codebook_values": 1384448,
"scales_changed": 32125070,
"scale_values": 49840128
}
},
{
"step": 55,
"validation": {
"target_rel": 0.2789926237064633,
"reference_rel": 0.2789926237064633
},
"reasoning_validation": null,
"training_probe_rel": 0.97875044237483,
"batch_loss": 0.9762788265943527,
"best_step": 15,
"overfit_checks": 0,
"seconds": 622.7157917022705,
"adaptive_schedule": {
"best": 0.2789341851468615,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 15,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 11564,
"codebook_values": 1384448,
"scales_changed": 32298881,
"scale_values": 49840128
}
},
{
"step": 60,
"validation": {
"target_rel": 0.2790563133795923,
"reference_rel": 0.2790563133795923
},
"reasoning_validation": null,
"training_probe_rel": 0.9788814159600921,
"batch_loss": 0.9402946084737778,
"best_step": 15,
"overfit_checks": 0,
"seconds": 690.7925577163696,
"adaptive_schedule": {
"best": 0.2789341851468615,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 15,
"early_drop": false
},
"adaptive_stop": true,
"quantized_movement": {
"codebook_changed": 12689,
"codebook_values": 1384448,
"scales_changed": 32458306,
"scale_values": 49840128
}
}
],
"index_reassignments": [
{
"method": "expert_parallel_output_gradient_prefix_backtracking_v1",
"proposals": 326,
"accepted": 265,
"changed_groups": 282106,
"training_before": 2392803.5,
"training_after": 2386653.5,
"acceptance": "proposal and separate training check batches",
"step": 20
},
{
"method": "expert_parallel_output_gradient_prefix_backtracking_v1",
"proposals": 332,
"accepted": 245,
"changed_groups": 265429,
"training_before": 2356026.0,
"training_after": 2354141.75,
"acceptance": "proposal and separate training check batches",
"step": 40
},
{
"method": "expert_parallel_output_gradient_prefix_backtracking_v1",
"proposals": 326,
"accepted": 256,
"changed_groups": 284234,
"training_before": 2296962.75,
"training_after": 2295092.0,
"acceptance": "proposal and separate training check batches",
"step": 60
}
],
"coverage_threshold": 256,
"checkpoint_selection": "held-out original reference trajectory for both arms",
"world_size": 8,
"microbatch": 65536,
"proposal_communication": "sparse routed rows",
"parallelism": "expert-sharded coupled output",
"arithmetic": "fp4_planes4_fp16_boundaries_v1",
"scope": "text-only, cached prompt partitions; historical audit is development only",
"train_rows": 18001846,
"validation_rows": 16384,
"audit_rows": 16384,
"complete": true
}