jarrelscy's picture
Publish GLM-5.3 NVFP4 / ARVQ v2 same_input fp16-block hybrid: non-cold structure seed (part 2)
868041e verified
Raw History Blame
22.6 kB
{
"layer": 40,
"target": "reference",
"recipe": "full_corpus_sequence_pass_v1",
"block_scale_dtype": "fp8_e4m3",
"batch_tokens": 262144,
"sequence_order_sha256": "edebb22a5f7b34a6cbfa59c58aab9f111651d6ce0926212b5237dea3ef0ba485",
"training_sequence_count": 17580,
"training_passes": 0.8737237281109949,
"training_tokens_seen": 15728640,
"adaptive_schedule": {
"best": 0.19522256129917523,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 45,
"early_drop": false
},
"refinement": null,
"lr_decay_after": 45,
"lr_decay_factor": 0.25,
"export_only": false,
"training_output_export": "fixed validation/audit outputs only; full training trajectory export not performed",
"codebook_lr": 0.048,
"scale_lr": 0.032,
"quantized_movement": {
"codebook_changed": 5230,
"codebook_values": 1482752,
"scales_changed": 36672020,
"scale_values": 53379072
},
"initial_validation": {
"target_rel": 0.1955733450864833,
"reference_rel": 0.1955733450864833,
"windows": [
{
"first_row": 0,
"rows": 1024,
"reference_sse": 49942.0234375,
"reference_energy": 1904173.75
},
{
"first_row": 1024,
"rows": 1024,
"reference_sse": 70319.21875,
"reference_energy": 1853071.5
},
{
"first_row": 4096,
"rows": 1024,
"reference_sse": 309096.71875,
"reference_energy": 3423241.5
},
{
"first_row": 5120,
"rows": 1024,
"reference_sse": 56463.2421875,
"reference_energy": 1858216.75
},
{
"first_row": 8192,
"rows": 1024,
"reference_sse": 156274.09375,
"reference_energy": 1849197.875
},
{
"first_row": 9216,
"rows": 1024,
"reference_sse": 60087.8359375,
"reference_energy": 5011509.0
},
{
"first_row": 12288,
"rows": 1024,
"reference_sse": 58230.2890625,
"reference_energy": 1872363.5
},
{
"first_row": 13312,
"rows": 1024,
"reference_sse": 46554.8828125,
"reference_energy": 1939973.0
},
{
"first_row": 16384,
"rows": 1024,
"reference_sse": 49746.97265625,
"reference_energy": 1929284.25
},
{
"first_row": 17408,
"rows": 1024,
"reference_sse": 50237.7109375,
"reference_energy": 1973334.875
},
{
"first_row": 20480,
"rows": 1024,
"reference_sse": 49161.3515625,
"reference_energy": 1972181.0
},
{
"first_row": 21504,
"rows": 1024,
"reference_sse": 80394.0,
"reference_energy": 3364985.25
},
{
"first_row": 24576,
"rows": 1024,
"reference_sse": 61179.38671875,
"reference_energy": 1934969.375
},
{
"first_row": 25600,
"rows": 1024,
"reference_sse": 40875.53125,
"reference_energy": 1953059.5
},
{
"first_row": 28672,
"rows": 1024,
"reference_sse": 44511.421875,
"reference_energy": 1830772.875
},
{
"first_row": 29696,
"rows": 1024,
"reference_sse": 216872.921875,
"reference_energy": 1930624.0
}
]
},
"final_validation": {
"target_rel": 0.19522256129917523,
"reference_rel": 0.19522256129917523,
"windows": [
{
"first_row": 0,
"rows": 1024,
"reference_sse": 49835.1953125,
"reference_energy": 1904173.75
},
{
"first_row": 1024,
"rows": 1024,
"reference_sse": 70165.65625,
"reference_energy": 1853071.5
},
{
"first_row": 4096,
"rows": 1024,
"reference_sse": 308541.125,
"reference_energy": 3423241.5
},
{
"first_row": 5120,
"rows": 1024,
"reference_sse": 56317.21875,
"reference_energy": 1858216.75
},
{
"first_row": 8192,
"rows": 1024,
"reference_sse": 154024.375,
"reference_energy": 1849197.875
},
{
"first_row": 9216,
"rows": 1024,
"reference_sse": 59745.01953125,
"reference_energy": 5011509.0
},
{
"first_row": 12288,
"rows": 1024,
"reference_sse": 58108.296875,
"reference_energy": 1872363.5
},
{
"first_row": 13312,
"rows": 1024,
"reference_sse": 46759.6875,
"reference_energy": 1939973.0
},
{
"first_row": 16384,
"rows": 1024,
"reference_sse": 49582.875,
"reference_energy": 1929284.25
},
{
"first_row": 17408,
"rows": 1024,
"reference_sse": 50120.0546875,
"reference_energy": 1973334.875
},
{
"first_row": 20480,
"rows": 1024,
"reference_sse": 49050.3046875,
"reference_energy": 1972181.0
},
{
"first_row": 21504,
"rows": 1024,
"reference_sse": 79713.4921875,
"reference_energy": 3364985.25
},
{
"first_row": 24576,
"rows": 1024,
"reference_sse": 61001.2578125,
"reference_energy": 1934969.375
},
{
"first_row": 25600,
"rows": 1024,
"reference_sse": 40790.53515625,
"reference_energy": 1953059.5
},
{
"first_row": 28672,
"rows": 1024,
"reference_sse": 44370.5078125,
"reference_energy": 1830772.875
},
{
"first_row": 29696,
"rows": 1024,
"reference_sse": 216804.5625,
"reference_energy": 1930624.0
}
]
},
"initial_reasoning_validation": null,
"final_reasoning_validation": null,
"validation_distribution": "reasoning only",
"serialized_validation": {
"target_rel": 0.19522256129917523,
"reference_rel": 0.19522256129917523
},
"development_audit": {
"target_rel": 0.1636333358036894,
"reference_rel": 0.1636333358036894,
"windows": [
{
"first_row": 2048,
"rows": 1024,
"reference_sse": 51291.39453125,
"reference_energy": 1879215.375
},
{
"first_row": 3072,
"rows": 1024,
"reference_sse": 60250.84375,
"reference_energy": 1987552.5
},
{
"first_row": 6144,
"rows": 1024,
"reference_sse": 46387.5,
"reference_energy": 1906448.0
},
{
"first_row": 7168,
"rows": 1024,
"reference_sse": 53855.9453125,
"reference_energy": 4964151.5
},
{
"first_row": 10240,
"rows": 1024,
"reference_sse": 68256.46875,
"reference_energy": 3420940.5
},
{
"first_row": 11264,
"rows": 1024,
"reference_sse": 44413.9921875,
"reference_energy": 1943767.5
},
{
"first_row": 14336,
"rows": 1024,
"reference_sse": 142155.109375,
"reference_energy": 3453174.75
},
{
"first_row": 15360,
"rows": 1024,
"reference_sse": 39980.1015625,
"reference_energy": 1971322.0
},
{
"first_row": 18432,
"rows": 1024,
"reference_sse": 55032.3671875,
"reference_energy": 1822063.875
},
{
"first_row": 19456,
"rows": 1024,
"reference_sse": 53349.4609375,
"reference_energy": 1919050.25
},
{
"first_row": 22528,
"rows": 1024,
"reference_sse": 49684.1171875,
"reference_energy": 1864245.0
},
{
"first_row": 23552,
"rows": 1024,
"reference_sse": 142906.796875,
"reference_energy": 1882592.875
},
{
"first_row": 26624,
"rows": 1024,
"reference_sse": 59948.05859375,
"reference_energy": 1900741.75
},
{
"first_row": 27648,
"rows": 1024,
"reference_sse": 55660.7890625,
"reference_energy": 3423835.0
},
{
"first_row": 30720,
"rows": 1024,
"reference_sse": 46146.9921875,
"reference_energy": 1839910.0
},
{
"first_row": 31744,
"rows": 1024,
"reference_sse": 49817.63671875,
"reference_energy": 1882782.375
}
]
},
"initial_development_audit": {
"target_rel": 0.16386855296880676,
"reference_rel": 0.16386855296880676,
"windows": [
{
"first_row": 2048,
"rows": 1024,
"reference_sse": 51397.453125,
"reference_energy": 1879215.375
},
{
"first_row": 3072,
"rows": 1024,
"reference_sse": 60357.71875,
"reference_energy": 1987552.5
},
{
"first_row": 6144,
"rows": 1024,
"reference_sse": 46449.9453125,
"reference_energy": 1906448.0
},
{
"first_row": 7168,
"rows": 1024,
"reference_sse": 54103.9296875,
"reference_energy": 4964151.5
},
{
"first_row": 10240,
"rows": 1024,
"reference_sse": 68553.40625,
"reference_energy": 3420940.5
},
{
"first_row": 11264,
"rows": 1024,
"reference_sse": 44212.4453125,
"reference_energy": 1943767.5
},
{
"first_row": 14336,
"rows": 1024,
"reference_sse": 143836.296875,
"reference_energy": 3453174.75
},
{
"first_row": 15360,
"rows": 1024,
"reference_sse": 39761.46875,
"reference_energy": 1971322.0
},
{
"first_row": 18432,
"rows": 1024,
"reference_sse": 55180.37890625,
"reference_energy": 1822063.875
},
{
"first_row": 19456,
"rows": 1024,
"reference_sse": 53477.5078125,
"reference_energy": 1919050.25
},
{
"first_row": 22528,
"rows": 1024,
"reference_sse": 49840.296875,
"reference_energy": 1864245.0
},
{
"first_row": 23552,
"rows": 1024,
"reference_sse": 143024.859375,
"reference_energy": 1882592.875
},
{
"first_row": 26624,
"rows": 1024,
"reference_sse": 60104.6484375,
"reference_energy": 1900741.75
},
{
"first_row": 27648,
"rows": 1024,
"reference_sse": 55594.6796875,
"reference_energy": 3423835.0
},
{
"first_row": 30720,
"rows": 1024,
"reference_sse": 46261.125,
"reference_energy": 1839910.0
},
{
"first_row": 31744,
"rows": 1024,
"reference_sse": 49913.46875,
"reference_energy": 1882782.375
}
]
},
"best_step": 45,
"steps_run": 60,
"history": [
{
"step": 5,
"validation": {
"target_rel": 0.19547191506608724,
"reference_rel": 0.19547191506608724
},
"reasoning_validation": null,
"training_probe_rel": 0.9564657231563846,
"batch_loss": 0.8945630788803101,
"best_step": 5,
"overfit_checks": 0,
"seconds": 51.58656620979309,
"adaptive_schedule": {
"best": 0.19547191506608724,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 5,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 0,
"codebook_values": 1482752,
"scales_changed": 10871072,
"scale_values": 53379072
}
},
{
"step": 10,
"validation": {
"target_rel": 0.19549133380791703,
"reference_rel": 0.19549133380791703
},
"reasoning_validation": null,
"training_probe_rel": 0.9559954787992033,
"batch_loss": 0.8914395719766617,
"best_step": 5,
"overfit_checks": 0,
"seconds": 108.8031976222992,
"adaptive_schedule": {
"best": 0.19547191506608724,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 5,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 33587,
"codebook_values": 1482752,
"scales_changed": 20592447,
"scale_values": 53379072
}
},
{
"step": 15,
"validation": {
"target_rel": 0.19541298177640284,
"reference_rel": 0.19541298177640284
},
"reasoning_validation": null,
"training_probe_rel": 0.9546498771273022,
"batch_loss": 0.9502743780612946,
"best_step": 15,
"overfit_checks": 0,
"seconds": 162.00546216964722,
"adaptive_schedule": {
"best": 0.19541298177640284,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 15,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 7829,
"codebook_values": 1482752,
"scales_changed": 25875140,
"scale_values": 53379072
}
},
{
"step": 20,
"validation": {
"target_rel": 0.19535734873993038,
"reference_rel": 0.19535734873993038
},
"reasoning_validation": null,
"training_probe_rel": 0.9539854898137821,
"batch_loss": 0.9207183867692947,
"best_step": 20,
"overfit_checks": 0,
"seconds": 230.0210530757904,
"adaptive_schedule": {
"best": 0.19535734873993038,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 20,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 6166,
"codebook_values": 1482752,
"scales_changed": 29314083,
"scale_values": 53379072
}
},
{
"step": 25,
"validation": {
"target_rel": 0.19532590835032326,
"reference_rel": 0.19532590835032326
},
"reasoning_validation": null,
"training_probe_rel": 0.9535948106849987,
"batch_loss": 0.9260809123516083,
"best_step": 25,
"overfit_checks": 0,
"seconds": 289.1424024105072,
"adaptive_schedule": {
"best": 0.19532590835032326,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 25,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 5446,
"codebook_values": 1482752,
"scales_changed": 31706314,
"scale_values": 53379072
}
},
{
"step": 30,
"validation": {
"target_rel": 0.1953112712116995,
"reference_rel": 0.1953112712116995
},
"reasoning_validation": null,
"training_probe_rel": 0.9533598515216203,
"batch_loss": 0.9161847531795502,
"best_step": 30,
"overfit_checks": 0,
"seconds": 349.34577441215515,
"adaptive_schedule": {
"best": 0.1953112712116995,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 30,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 4859,
"codebook_values": 1482752,
"scales_changed": 33449235,
"scale_values": 53379072
}
},
{
"step": 35,
"validation": {
"target_rel": 0.1952900185653858,
"reference_rel": 0.1952900185653858
},
"reasoning_validation": null,
"training_probe_rel": 0.9531974392435874,
"batch_loss": 0.9673894643783569,
"best_step": 35,
"overfit_checks": 0,
"seconds": 402.4638192653656,
"adaptive_schedule": {
"best": 0.1952900185653858,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 35,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 4668,
"codebook_values": 1482752,
"scales_changed": 34775582,
"scale_values": 53379072
}
},
{
"step": 40,
"validation": {
"target_rel": 0.1952627503023498,
"reference_rel": 0.1952627503023498
},
"reasoning_validation": null,
"training_probe_rel": 0.9531078328070768,
"batch_loss": 0.876895472407341,
"best_step": 40,
"overfit_checks": 0,
"seconds": 475.6668243408203,
"adaptive_schedule": {
"best": 0.1952627503023498,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 40,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 4940,
"codebook_values": 1482752,
"scales_changed": 35827238,
"scale_values": 53379072
}
},
{
"step": 45,
"validation": {
"target_rel": 0.19522256129917523,
"reference_rel": 0.19522256129917523
},
"reasoning_validation": null,
"training_probe_rel": 0.9530780689521952,
"batch_loss": 0.8754175752401352,
"best_step": 45,
"overfit_checks": 0,
"seconds": 528.7454051971436,
"adaptive_schedule": {
"best": 0.19522256129917523,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 45,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 5230,
"codebook_values": 1482752,
"scales_changed": 36672020,
"scale_values": 53379072
}
},
{
"step": 50,
"validation": {
"target_rel": 0.1952339424133861,
"reference_rel": 0.1952339424133861
},
"reasoning_validation": null,
"training_probe_rel": 0.9528936142200086,
"batch_loss": 0.9313409328460693,
"best_step": 45,
"overfit_checks": 0,
"seconds": 586.7400496006012,
"adaptive_schedule": {
"best": 0.19522256129917523,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 45,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 2687,
"codebook_values": 1482752,
"scales_changed": 36841723,
"scale_values": 53379072
}
},
{
"step": 55,
"validation": {
"target_rel": 0.1952467192045548,
"reference_rel": 0.1952467192045548
},
"reasoning_validation": null,
"training_probe_rel": 0.9528730905608923,
"batch_loss": 0.9879082590341568,
"best_step": 45,
"overfit_checks": 0,
"seconds": 640.1329443454742,
"adaptive_schedule": {
"best": 0.19522256129917523,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 45,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 2823,
"codebook_values": 1482752,
"scales_changed": 36984762,
"scale_values": 53379072
}
},
{
"step": 60,
"validation": {
"target_rel": 0.19524598864838574,
"reference_rel": 0.19524598864838574
},
"reasoning_validation": null,
"training_probe_rel": 0.9528624134805262,
"batch_loss": 0.9161676466464996,
"best_step": 45,
"overfit_checks": 0,
"seconds": 704.2472217082977,
"adaptive_schedule": {
"best": 0.19522256129917523,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 45,
"early_drop": false
},
"adaptive_stop": true,
"quantized_movement": {
"codebook_changed": 3097,
"codebook_values": 1482752,
"scales_changed": 37111700,
"scale_values": 53379072
}
}
],
"index_reassignments": [
{
"method": "expert_parallel_output_gradient_prefix_backtracking_v1",
"proposals": 354,
"accepted": 293,
"changed_groups": 179992,
"training_before": 719517.25,
"training_after": 718894.0,
"acceptance": "proposal and separate training check batches",
"step": 20
},
{
"method": "expert_parallel_output_gradient_prefix_backtracking_v1",
"proposals": 352,
"accepted": 325,
"changed_groups": 207248,
"training_before": 629022.625,
"training_after": 628485.125,
"acceptance": "proposal and separate training check batches",
"step": 40
},
{
"method": "expert_parallel_output_gradient_prefix_backtracking_v1",
"proposals": 354,
"accepted": 307,
"changed_groups": 198945,
"training_before": 651169.375,
"training_after": 650657.75,
"acceptance": "proposal and separate training check batches",
"step": 60
}
],
"coverage_threshold": 256,
"checkpoint_selection": "held-out original reference trajectory for both arms",
"world_size": 8,
"microbatch": 65536,
"proposal_communication": "sparse routed rows",
"parallelism": "expert-sharded coupled output",
"arithmetic": "fp4_planes4_fp16_boundaries_v1",
"scope": "text-only, cached prompt partitions; historical audit is development only",
"train_rows": 18001846,
"validation_rows": 16384,
"audit_rows": 16384,
"complete": true
}