jarrelscy's picture
Publish GLM-5.3 NVFP4 / ARVQ v2 same_input fp16-block hybrid: non-cold structure seed (part 2)
868041e verified
Raw History Blame
22.7 kB
{
"layer": 32,
"target": "reference",
"recipe": "full_corpus_sequence_pass_v1",
"batch_tokens": 262144,
"sequence_order_sha256": "edebb22a5f7b34a6cbfa59c58aab9f111651d6ce0926212b5237dea3ef0ba485",
"training_sequence_count": 17580,
"training_passes": 0.8737237281109949,
"training_tokens_seen": 15728640,
"adaptive_schedule": {
"best": 0.13401018917728869,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 40,
"early_drop": false
},
"refinement": null,
"lr_decay_after": 45,
"lr_decay_factor": 0.25,
"export_only": false,
"training_output_export": "fixed validation/audit outputs only; full training trajectory export not performed",
"codebook_lr": 0.048,
"scale_lr": 0.032,
"quantized_movement": {
"codebook_changed": 6354,
"codebook_values": 1441792,
"scales_changed": 34163419,
"scale_values": 51904512
},
"initial_validation": {
"target_rel": 0.13427382523869696,
"reference_rel": 0.13427382523869696,
"windows": [
{
"first_row": 0,
"rows": 1024,
"reference_sse": 4890.314453125,
"reference_energy": 1460955.5
},
{
"first_row": 1024,
"rows": 1024,
"reference_sse": 13987.2509765625,
"reference_energy": 1415606.125
},
{
"first_row": 4096,
"rows": 1024,
"reference_sse": 195478.375,
"reference_energy": 2904194.0
},
{
"first_row": 5120,
"rows": 1024,
"reference_sse": 5472.20849609375,
"reference_energy": 1425121.75
},
{
"first_row": 8192,
"rows": 1024,
"reference_sse": 100947.3125,
"reference_energy": 1399778.25
},
{
"first_row": 9216,
"rows": 1024,
"reference_sse": 8410.123046875,
"reference_energy": 4361363.0
},
{
"first_row": 12288,
"rows": 1024,
"reference_sse": 5827.9501953125,
"reference_energy": 1421753.75
},
{
"first_row": 13312,
"rows": 1024,
"reference_sse": 4099.7978515625,
"reference_energy": 1499537.5
},
{
"first_row": 16384,
"rows": 1024,
"reference_sse": 5228.93505859375,
"reference_energy": 1481459.5
},
{
"first_row": 17408,
"rows": 1024,
"reference_sse": 5910.96533203125,
"reference_energy": 1527533.875
},
{
"first_row": 20480,
"rows": 1024,
"reference_sse": 4633.908203125,
"reference_energy": 1514512.875
},
{
"first_row": 21504,
"rows": 1024,
"reference_sse": 10089.212890625,
"reference_energy": 2854828.5
},
{
"first_row": 24576,
"rows": 1024,
"reference_sse": 5300.84375,
"reference_energy": 1501097.375
},
{
"first_row": 25600,
"rows": 1024,
"reference_sse": 4295.76220703125,
"reference_energy": 1503226.125
},
{
"first_row": 28672,
"rows": 1024,
"reference_sse": 5834.88818359375,
"reference_energy": 1392998.5
},
{
"first_row": 29696,
"rows": 1024,
"reference_sse": 145143.6875,
"reference_energy": 1485632.5
}
]
},
"final_validation": {
"target_rel": 0.13401018917728869,
"reference_rel": 0.13401018917728869,
"windows": [
{
"first_row": 0,
"rows": 1024,
"reference_sse": 4888.134765625,
"reference_energy": 1460955.5
},
{
"first_row": 1024,
"rows": 1024,
"reference_sse": 13967.861328125,
"reference_energy": 1415606.125
},
{
"first_row": 4096,
"rows": 1024,
"reference_sse": 195307.796875,
"reference_energy": 2904194.0
},
{
"first_row": 5120,
"rows": 1024,
"reference_sse": 5471.0849609375,
"reference_energy": 1425121.75
},
{
"first_row": 8192,
"rows": 1024,
"reference_sse": 98979.3671875,
"reference_energy": 1399778.25
},
{
"first_row": 9216,
"rows": 1024,
"reference_sse": 8328.857421875,
"reference_energy": 4361363.0
},
{
"first_row": 12288,
"rows": 1024,
"reference_sse": 5826.421875,
"reference_energy": 1421753.75
},
{
"first_row": 13312,
"rows": 1024,
"reference_sse": 4141.560546875,
"reference_energy": 1499537.5
},
{
"first_row": 16384,
"rows": 1024,
"reference_sse": 5346.01806640625,
"reference_energy": 1481459.5
},
{
"first_row": 17408,
"rows": 1024,
"reference_sse": 5937.76416015625,
"reference_energy": 1527533.875
},
{
"first_row": 20480,
"rows": 1024,
"reference_sse": 4670.0732421875,
"reference_energy": 1514512.875
},
{
"first_row": 21504,
"rows": 1024,
"reference_sse": 10015.8203125,
"reference_energy": 2854828.5
},
{
"first_row": 24576,
"rows": 1024,
"reference_sse": 5322.6708984375,
"reference_energy": 1501097.375
},
{
"first_row": 25600,
"rows": 1024,
"reference_sse": 4388.23046875,
"reference_energy": 1503226.125
},
{
"first_row": 28672,
"rows": 1024,
"reference_sse": 5823.8916015625,
"reference_energy": 1392998.5
},
{
"first_row": 29696,
"rows": 1024,
"reference_sse": 145074.25,
"reference_energy": 1485632.5
}
]
},
"initial_reasoning_validation": null,
"final_reasoning_validation": null,
"validation_distribution": "reasoning only",
"serialized_validation": {
"target_rel": 0.13401018917728869,
"reference_rel": 0.13401018917728869
},
"development_audit": {
"target_rel": 0.0895711648625643,
"reference_rel": 0.0895711648625643,
"windows": [
{
"first_row": 2048,
"rows": 1024,
"reference_sse": 5575.583984375,
"reference_energy": 1429303.75
},
{
"first_row": 3072,
"rows": 1024,
"reference_sse": 7218.240234375,
"reference_energy": 1535322.75
},
{
"first_row": 6144,
"rows": 1024,
"reference_sse": 4985.3427734375,
"reference_energy": 1456362.0
},
{
"first_row": 7168,
"rows": 1024,
"reference_sse": 7697.3720703125,
"reference_energy": 4341793.0
},
{
"first_row": 10240,
"rows": 1024,
"reference_sse": 7231.7939453125,
"reference_energy": 2908677.5
},
{
"first_row": 11264,
"rows": 1024,
"reference_sse": 5927.826171875,
"reference_energy": 1502530.5
},
{
"first_row": 14336,
"rows": 1024,
"reference_sse": 70326.28125,
"reference_energy": 2932214.25
},
{
"first_row": 15360,
"rows": 1024,
"reference_sse": 4273.09326171875,
"reference_energy": 1533464.875
},
{
"first_row": 18432,
"rows": 1024,
"reference_sse": 5231.751953125,
"reference_energy": 1378049.25
},
{
"first_row": 19456,
"rows": 1024,
"reference_sse": 5569.607421875,
"reference_energy": 1493604.25
},
{
"first_row": 22528,
"rows": 1024,
"reference_sse": 5351.43359375,
"reference_energy": 1436936.0
},
{
"first_row": 23552,
"rows": 1024,
"reference_sse": 91289.046875,
"reference_energy": 1442902.125
},
{
"first_row": 26624,
"rows": 1024,
"reference_sse": 5104.244140625,
"reference_energy": 1452026.25
},
{
"first_row": 27648,
"rows": 1024,
"reference_sse": 6250.7548828125,
"reference_energy": 2916719.5
},
{
"first_row": 30720,
"rows": 1024,
"reference_sse": 4619.5966796875,
"reference_energy": 1391255.75
},
{
"first_row": 31744,
"rows": 1024,
"reference_sse": 8891.6337890625,
"reference_energy": 1453823.875
}
]
},
"initial_development_audit": {
"target_rel": 0.0898691005608009,
"reference_rel": 0.0898691005608009,
"windows": [
{
"first_row": 2048,
"rows": 1024,
"reference_sse": 5576.53466796875,
"reference_energy": 1429303.75
},
{
"first_row": 3072,
"rows": 1024,
"reference_sse": 7172.20166015625,
"reference_energy": 1535322.75
},
{
"first_row": 6144,
"rows": 1024,
"reference_sse": 4991.36572265625,
"reference_energy": 1456362.0
},
{
"first_row": 7168,
"rows": 1024,
"reference_sse": 7257.412109375,
"reference_energy": 4341793.0
},
{
"first_row": 10240,
"rows": 1024,
"reference_sse": 7251.1669921875,
"reference_energy": 2908677.5
},
{
"first_row": 11264,
"rows": 1024,
"reference_sse": 5904.55224609375,
"reference_energy": 1502530.5
},
{
"first_row": 14336,
"rows": 1024,
"reference_sse": 71298.203125,
"reference_energy": 2932214.25
},
{
"first_row": 15360,
"rows": 1024,
"reference_sse": 4204.484375,
"reference_energy": 1533464.875
},
{
"first_row": 18432,
"rows": 1024,
"reference_sse": 5237.42333984375,
"reference_energy": 1378049.25
},
{
"first_row": 19456,
"rows": 1024,
"reference_sse": 5576.4052734375,
"reference_energy": 1493604.25
},
{
"first_row": 22528,
"rows": 1024,
"reference_sse": 5309.66748046875,
"reference_energy": 1436936.0
},
{
"first_row": 23552,
"rows": 1024,
"reference_sse": 91901.1640625,
"reference_energy": 1442902.125
},
{
"first_row": 26624,
"rows": 1024,
"reference_sse": 5051.525390625,
"reference_energy": 1452026.25
},
{
"first_row": 27648,
"rows": 1024,
"reference_sse": 6115.37646484375,
"reference_energy": 2916719.5
},
{
"first_row": 30720,
"rows": 1024,
"reference_sse": 4630.05859375,
"reference_energy": 1391255.75
},
{
"first_row": 31744,
"rows": 1024,
"reference_sse": 9702.2548828125,
"reference_energy": 1453823.875
}
]
},
"best_step": 40,
"steps_run": 60,
"history": [
{
"step": 5,
"validation": {
"target_rel": 0.13421345428892492,
"reference_rel": 0.13421345428892492
},
"reasoning_validation": null,
"training_probe_rel": 0.8857626125183489,
"batch_loss": 0.789747029542923,
"best_step": 5,
"overfit_checks": 0,
"seconds": 56.834206342697144,
"adaptive_schedule": {
"best": 0.13421345428892492,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 5,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 0,
"codebook_values": 1441792,
"scales_changed": 10456178,
"scale_values": 51904512
}
},
{
"step": 10,
"validation": {
"target_rel": 0.13419196973060385,
"reference_rel": 0.13419196973060385
},
"reasoning_validation": null,
"training_probe_rel": 0.8859456687993534,
"batch_loss": 0.7986292243003845,
"best_step": 10,
"overfit_checks": 0,
"seconds": 110.04065608978271,
"adaptive_schedule": {
"best": 0.13419196973060385,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 10,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 18583,
"codebook_values": 1441792,
"scales_changed": 19862552,
"scale_values": 51904512
}
},
{
"step": 15,
"validation": {
"target_rel": 0.13413925650239486,
"reference_rel": 0.13413925650239486
},
"reasoning_validation": null,
"training_probe_rel": 0.88399760976569,
"batch_loss": 0.9806730896234512,
"best_step": 15,
"overfit_checks": 0,
"seconds": 162.61236429214478,
"adaptive_schedule": {
"best": 0.13413925650239486,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 15,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 6478,
"codebook_values": 1441792,
"scales_changed": 25090108,
"scale_values": 51904512
}
},
{
"step": 20,
"validation": {
"target_rel": 0.13409019432159866,
"reference_rel": 0.13409019432159866
},
"reasoning_validation": null,
"training_probe_rel": 0.8835375299840819,
"batch_loss": 0.8944355100393295,
"best_step": 20,
"overfit_checks": 0,
"seconds": 236.87151861190796,
"adaptive_schedule": {
"best": 0.13409019432159866,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 20,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 6487,
"codebook_values": 1441792,
"scales_changed": 28341744,
"scale_values": 51904512
}
},
{
"step": 25,
"validation": {
"target_rel": 0.13409478857774693,
"reference_rel": 0.13409478857774693
},
"reasoning_validation": null,
"training_probe_rel": 0.8829107627031227,
"batch_loss": 0.859352707862854,
"best_step": 20,
"overfit_checks": 0,
"seconds": 295.64807391166687,
"adaptive_schedule": {
"best": 0.13409019432159866,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 20,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 5993,
"codebook_values": 1441792,
"scales_changed": 30506606,
"scale_values": 51904512
}
},
{
"step": 30,
"validation": {
"target_rel": 0.1341149755184966,
"reference_rel": 0.1341149755184966
},
"reasoning_validation": null,
"training_probe_rel": 0.8828862989671643,
"batch_loss": 0.825282409787178,
"best_step": 20,
"overfit_checks": 0,
"seconds": 349.9026427268982,
"adaptive_schedule": {
"best": 0.13409019432159866,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 20,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 5682,
"codebook_values": 1441792,
"scales_changed": 32054741,
"scale_values": 51904512
}
},
{
"step": 35,
"validation": {
"target_rel": 0.13409664897092413,
"reference_rel": 0.13409664897092413
},
"reasoning_validation": null,
"training_probe_rel": 0.8825478131304417,
"batch_loss": 0.9860599488019943,
"best_step": 20,
"overfit_checks": 0,
"seconds": 411.4317955970764,
"adaptive_schedule": {
"best": 0.13409019432159866,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 20,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 5429,
"codebook_values": 1441792,
"scales_changed": 33229748,
"scale_values": 51904512
}
},
{
"step": 40,
"validation": {
"target_rel": 0.13401018917728869,
"reference_rel": 0.13401018917728869
},
"reasoning_validation": null,
"training_probe_rel": 0.8833369747955293,
"batch_loss": 0.7437251955270767,
"best_step": 40,
"overfit_checks": 0,
"seconds": 481.8696639537811,
"adaptive_schedule": {
"best": 0.13401018917728869,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 40,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 6354,
"codebook_values": 1441792,
"scales_changed": 34163419,
"scale_values": 51904512
}
},
{
"step": 45,
"validation": {
"target_rel": 0.13405527457873018,
"reference_rel": 0.13405527457873018
},
"reasoning_validation": null,
"training_probe_rel": 0.882505296500861,
"batch_loss": 0.7459392100572586,
"best_step": 40,
"overfit_checks": 0,
"seconds": 541.5856935977936,
"adaptive_schedule": {
"best": 0.13401018917728869,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 40,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 6674,
"codebook_values": 1441792,
"scales_changed": 34907593,
"scale_values": 51904512
}
},
{
"step": 50,
"validation": {
"target_rel": 0.13406860929796122,
"reference_rel": 0.13406860929796122
},
"reasoning_validation": null,
"training_probe_rel": 0.8817115940464041,
"batch_loss": 0.8900789767503738,
"best_step": 40,
"overfit_checks": 0,
"seconds": 595.977490901947,
"adaptive_schedule": {
"best": 0.13401018917728869,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 40,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 4974,
"codebook_values": 1441792,
"scales_changed": 35054620,
"scale_values": 51904512
}
},
{
"step": 55,
"validation": {
"target_rel": 0.13407398415614327,
"reference_rel": 0.13407398415614327
},
"reasoning_validation": null,
"training_probe_rel": 0.8819151892489847,
"batch_loss": 1.0232778787612915,
"best_step": 40,
"overfit_checks": 0,
"seconds": 654.4850785732269,
"adaptive_schedule": {
"best": 0.13401018917728869,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 40,
"early_drop": false
},
"adaptive_stop": false,
"quantized_movement": {
"codebook_changed": 5303,
"codebook_values": 1441792,
"scales_changed": 35174246,
"scale_values": 51904512
}
},
{
"step": 60,
"validation": {
"target_rel": 0.13408069288425276,
"reference_rel": 0.13408069288425276
},
"reasoning_validation": null,
"training_probe_rel": 0.8818878248991407,
"batch_loss": 0.8886172026395798,
"best_step": 40,
"overfit_checks": 0,
"seconds": 722.5865497589111,
"adaptive_schedule": {
"best": 0.13401018917728869,
"drop_after": 45,
"relative_worsening": 0.001,
"checks": 3,
"patience_updates": 15,
"bad_checks": 0,
"last_best": 40,
"early_drop": false
},
"adaptive_stop": true,
"quantized_movement": {
"codebook_changed": 5114,
"codebook_values": 1441792,
"scales_changed": 35273861,
"scale_values": 51904512
}
}
],
"index_reassignments": [
{
"method": "expert_parallel_output_gradient_prefix_backtracking_v1",
"proposals": 344,
"accepted": 246,
"changed_groups": 173625,
"training_before": 221754.9375,
"training_after": 221596.21875,
"acceptance": "proposal and separate training check batches",
"step": 20
},
{
"method": "expert_parallel_output_gradient_prefix_backtracking_v1",
"proposals": 346,
"accepted": 327,
"changed_groups": 203118,
"training_before": 136497.59375,
"training_after": 134817.671875,
"acceptance": "proposal and separate training check batches",
"step": 40
},
{
"method": "expert_parallel_output_gradient_prefix_backtracking_v1",
"proposals": 344,
"accepted": 314,
"changed_groups": 219457,
"training_before": 168905.8125,
"training_after": 168788.0625,
"acceptance": "proposal and separate training check batches",
"step": 60
}
],
"coverage_threshold": 256,
"checkpoint_selection": "held-out original reference trajectory for both arms",
"world_size": 8,
"microbatch": 65536,
"proposal_communication": "sparse routed rows",
"parallelism": "expert-sharded coupled output",
"arithmetic": "fp4_planes4_fp16_boundaries_v1",
"scope": "text-only, cached prompt partitions; historical audit is development only",
"train_rows": 18001846,
"validation_rows": 16384,
"audit_rows": 16384,
"complete": true
}