{ "layer": 16, "cold_expert_ids": [ 1, 2, 4, 5, 7, 8, 9, 10, 11, 13, 14, 15, 18, 20, 23, 26, 28, 32, 34, 36, 37, 39, 40, 41, 42, 43, 44, 46, 47, 50, 51, 52, 54, 57, 58, 60, 63, 64, 65, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 79, 80, 81, 83, 85, 86, 88, 89, 90, 91, 92, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 109, 110, 111, 112, 114, 116, 117, 118, 119, 121, 122, 124, 125, 126, 127, 128, 129, 131, 133, 134, 135, 136, 138, 140, 141, 142, 143, 145, 147, 149, 150, 151, 152, 156, 157, 158, 159, 160, 161, 162, 164, 167, 168, 170, 171, 172, 173, 174, 175, 177, 178, 179, 181, 182, 186, 188, 189, 192, 195, 197, 198, 200, 202, 204, 207, 209, 210, 212, 214, 216, 217, 220, 221, 223, 226, 227, 228, 232, 234, 235, 236, 237, 238, 241, 243, 245, 248, 250, 252, 253, 254 ], "files": { "w13": { "file": "arvq-layer-016-gateup.safetensors", "sha256": "7d7414cd8b0050c79f47eb60805fabca496affed996e8ccd7283adb6b0cf96c3" }, "w2": { "file": "arvq-layer-016-down.safetensors", "sha256": "3b45c869739bf9d467a46f7e8d7febe0cadf7e3733f69a8fb1cdf199ad2425bc" } }, "per_proj": { "w13": { "cb_rel_l2": 0.3404741435598547, "hess_recon_rel_fro": 0.147139759333124, "diagnostic_metric": "regularized fitting Hessian; mean expert relative error for expert scope", "seconds": 310.29145336151123 }, "w2": { "cb_rel_l2": 0.3399461410924665, "hess_recon_rel_fro": 0.23373660962169931, "diagnostic_metric": "regularized fitting Hessian; mean expert relative error for expert scope", "seconds": 105.86656856536865 } }, "no_rotation": true, "source_identity": { "source": { "repo": "zai-org/GLM-5.3", "revision": "aca966e4e02791568aa6a4ced368624b3d897f42", "meta": { "kind": "block_fp8", "block": [ 128, 128 ], "fmt": "e4m3" }, "cache": "/tmp/glm53-fp8-cold", "legacy_cache_revision_unverified": true }, "capture_sha256": "f62ac7e0a71ef8d17ae63244801652d30d80439095aa2e1e3d5f46a7f80b2b3e", "cold_ids": [ 1, 2, 4, 5, 7, 8, 9, 10, 11, 13, 14, 15, 18, 20, 23, 26, 28, 32, 34, 36, 37, 39, 40, 41, 42, 43, 44, 46, 47, 50, 51, 52, 54, 57, 58, 60, 63, 64, 65, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 79, 80, 81, 83, 85, 86, 88, 89, 90, 91, 92, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 109, 110, 111, 112, 114, 116, 117, 118, 119, 121, 122, 124, 125, 126, 127, 128, 129, 131, 133, 134, 135, 136, 138, 140, 141, 142, 143, 145, 147, 149, 150, 151, 152, 156, 157, 158, 159, 160, 161, 162, 164, 167, 168, 170, 171, 172, 173, 174, 175, 177, 178, 179, 181, 182, 186, 188, 189, 192, 195, 197, 198, 200, 202, 204, 207, 209, 210, 212, 214, 216, 217, 220, 221, 223, 226, 227, 228, 232, 234, 235, 236, 237, 238, 241, 243, 245, 248, 250, 252, 253, 254 ], "encoder_sha256": "6c20c56fc7a308724ab4faf4174c4e8e32cba5a43c3fccd3f3e6d4da1f0b583e", "settings": { "cb_iters": 8, "sweep_passes": 2, "refine": 1, "col_block": 128, "codebook_scope": "expert" } }, "format": "rvq256_256x8_expert_fp16block", "fit": "sequential same_input-trajectory gradient PV; recipe=full_corpus_sequence_pass_v1; updates=30", "pv_report": { "layer": 16, "target": "same_input", "recipe": "full_corpus_sequence_pass_v1", "block_scale_dtype": "fp16", "batch_tokens": 262144, "sequence_order_sha256": "5e0ef183802009d0483fe37a60f5acd49dd99218130e0c952f097fa473d863bf", "training_sequence_count": 14657, "training_passes": 0.5240171180844249, "training_tokens_seen": 7864320, "adaptive_schedule": { "best": 0.004101365709508885, "drop_after": 20, "relative_worsening": 0.001, "checks": 3, "patience_updates": 10, "bad_checks": 3, "last_best": 5, "early_drop": true }, "refinement": null, "lr_decay_after": 25, "lr_decay_factor": 0.25, "export_only": false, "training_output_export": "fixed validation/audit outputs only; full training trajectory export not performed", "codebook_lr": 0.048, "scale_lr": 0.032, "quantized_movement": { "codebook_changed": 0, "codebook_values": 1359872, "scales_changed": 48733498, "scale_values": 48955392 }, "initial_validation": { "target_rel": 0.004312830133914392, "reference_rel": 0.0124293411154448, "boundary_rel": 0.0866977327475872, "bulk_rel": 0.004306373908448661, "boundary_rows": 24, "windows": [ { "first_row": 8192, "rows": 1024, "reference_sse": 109.70895385742188, "reference_energy": 833773.875 }, { "first_row": 9216, "rows": 1024, "reference_sse": 74.70093536376953, "reference_energy": 404147.78125 }, { "first_row": 20480, "rows": 1024, "reference_sse": 80.70475769042969, "reference_energy": 830239.875 }, { "first_row": 21504, "rows": 1024, "reference_sse": 82.34074401855469, "reference_energy": 366951.4375 }, { "first_row": 32768, "rows": 1024, "reference_sse": 90.87347412109375, "reference_energy": 831653.9375 }, { "first_row": 33792, "rows": 1024, "reference_sse": 102.24400329589844, "reference_energy": 370655.5625 }, { "first_row": 45056, "rows": 1024, "reference_sse": 91.02272033691406, "reference_energy": 841013.0 }, { "first_row": 46080, "rows": 1024, "reference_sse": 124.834228515625, "reference_energy": 815440.375 }, { "first_row": 57344, "rows": 1024, "reference_sse": 137.20108032226562, "reference_energy": 839627.9375 }, { "first_row": 58368, "rows": 1024, "reference_sse": 76.3934326171875, "reference_energy": 842679.375 }, { "first_row": 69632, "rows": 1024, "reference_sse": 95.2065200805664, "reference_energy": 821235.4375 }, { "first_row": 70656, "rows": 1024, "reference_sse": 87.90850067138672, "reference_energy": 402306.03125 }, { "first_row": 81920, "rows": 1024, "reference_sse": 91.58096313476562, "reference_energy": 822688.75 }, { "first_row": 82944, "rows": 1024, "reference_sse": 100.51765441894531, "reference_energy": 399803.6875 }, { "first_row": 94208, "rows": 1024, "reference_sse": 136.05613708496094, "reference_energy": 822497.875 }, { "first_row": 95232, "rows": 1024, "reference_sse": 164.42474365234375, "reference_energy": 407978.5 } ] }, "final_validation": { "target_rel": 0.004101365709508885, "reference_rel": 0.012358429863239363, "boundary_rel": 0.0845825288851992, "bulk_rel": 0.004094902819067128, "boundary_rows": 24, "windows": [ { "first_row": 8192, "rows": 1024, "reference_sse": 108.45175170898438, "reference_energy": 833773.875 }, { "first_row": 9216, "rows": 1024, "reference_sse": 74.10397338867188, "reference_energy": 404147.78125 }, { "first_row": 20480, "rows": 1024, "reference_sse": 79.19939422607422, "reference_energy": 830239.875 }, { "first_row": 21504, "rows": 1024, "reference_sse": 81.28684997558594, "reference_energy": 366951.4375 }, { "first_row": 32768, "rows": 1024, "reference_sse": 90.3306884765625, "reference_energy": 831653.9375 }, { "first_row": 33792, "rows": 1024, "reference_sse": 101.43246459960938, "reference_energy": 370655.5625 }, { "first_row": 45056, "rows": 1024, "reference_sse": 89.44915771484375, "reference_energy": 841013.0 }, { "first_row": 46080, "rows": 1024, "reference_sse": 124.42677307128906, "reference_energy": 815440.375 }, { "first_row": 57344, "rows": 1024, "reference_sse": 135.75778198242188, "reference_energy": 839627.9375 }, { "first_row": 58368, "rows": 1024, "reference_sse": 75.20976257324219, "reference_energy": 842679.375 }, { "first_row": 69632, "rows": 1024, "reference_sse": 93.64247131347656, "reference_energy": 821235.4375 }, { "first_row": 70656, "rows": 1024, "reference_sse": 86.43704223632812, "reference_energy": 402306.03125 }, { "first_row": 81920, "rows": 1024, "reference_sse": 90.456787109375, "reference_energy": 822688.75 }, { "first_row": 82944, "rows": 1024, "reference_sse": 99.3521728515625, "reference_energy": 399803.6875 }, { "first_row": 94208, "rows": 1024, "reference_sse": 135.04473876953125, "reference_energy": 822497.875 }, { "first_row": 95232, "rows": 1024, "reference_sse": 162.41246032714844, "reference_energy": 407978.5 } ] }, "initial_reasoning_validation": null, "final_reasoning_validation": null, "validation_distribution": "reasoning only", "serialized_validation": { "target_rel": 0.004101365709508885, "reference_rel": 0.012358429863239363 }, "development_audit": { "target_rel": 0.0060937046648625835, "reference_rel": 0.019080420843385917, "boundary_rel": 0.1208664018662727, "bulk_rel": 0.0060886062234531335, "boundary_rows": 10, "windows": [ { "first_row": 10240, "rows": 1024, "reference_sse": 351.2093505859375, "reference_energy": 799558.25 }, { "first_row": 11264, "rows": 1024, "reference_sse": 275.8849792480469, "reference_energy": 833834.5 }, { "first_row": 22528, "rows": 1024, "reference_sse": 330.35797119140625, "reference_energy": 405128.8125 }, { "first_row": 23552, "rows": 1024, "reference_sse": 230.50341796875, "reference_energy": 398648.21875 }, { "first_row": 34816, "rows": 1024, "reference_sse": 260.7713928222656, "reference_energy": 828490.0625 }, { "first_row": 35840, "rows": 1024, "reference_sse": 225.38717651367188, "reference_energy": 834319.875 }, { "first_row": 47104, "rows": 1024, "reference_sse": 223.26947021484375, "reference_energy": 380186.375 }, { "first_row": 48128, "rows": 1024, "reference_sse": 203.45504760742188, "reference_energy": 401168.71875 }, { "first_row": 59392, "rows": 1024, "reference_sse": 244.65362548828125, "reference_energy": 807945.5 }, { "first_row": 60416, "rows": 1024, "reference_sse": 199.71282958984375, "reference_energy": 816124.0625 }, { "first_row": 71680, "rows": 1024, "reference_sse": 215.0964813232422, "reference_energy": 825268.6875 }, { "first_row": 72704, "rows": 1024, "reference_sse": 194.43630981445312, "reference_energy": 833096.25 }, { "first_row": 83968, "rows": 1024, "reference_sse": 222.4454345703125, "reference_energy": 823285.9375 }, { "first_row": 84992, "rows": 1024, "reference_sse": 262.431640625, "reference_energy": 388802.125 }, { "first_row": 96256, "rows": 1024, "reference_sse": 208.48577880859375, "reference_energy": 396545.9375 }, { "first_row": 97280, "rows": 1024, "reference_sse": 213.8697509765625, "reference_energy": 835585.9375 } ] }, "initial_development_audit": { "target_rel": 0.0062904611388534095, "reference_rel": 0.019197126463551144, "boundary_rel": 0.12471609276894911, "bulk_rel": 0.006285202604186519, "boundary_rows": 10, "windows": [ { "first_row": 10240, "rows": 1024, "reference_sse": 353.823486328125, "reference_energy": 799558.25 }, { "first_row": 11264, "rows": 1024, "reference_sse": 278.6612548828125, "reference_energy": 833834.5 }, { "first_row": 22528, "rows": 1024, "reference_sse": 333.88909912109375, "reference_energy": 405128.8125 }, { "first_row": 23552, "rows": 1024, "reference_sse": 233.82662963867188, "reference_energy": 398648.21875 }, { "first_row": 34816, "rows": 1024, "reference_sse": 263.35552978515625, "reference_energy": 828490.0625 }, { "first_row": 35840, "rows": 1024, "reference_sse": 227.351318359375, "reference_energy": 834319.875 }, { "first_row": 47104, "rows": 1024, "reference_sse": 226.82522583007812, "reference_energy": 380186.375 }, { "first_row": 48128, "rows": 1024, "reference_sse": 206.85894775390625, "reference_energy": 401168.71875 }, { "first_row": 59392, "rows": 1024, "reference_sse": 247.02291870117188, "reference_energy": 807945.5 }, { "first_row": 60416, "rows": 1024, "reference_sse": 201.81112670898438, "reference_energy": 816124.0625 }, { "first_row": 71680, "rows": 1024, "reference_sse": 219.43948364257812, "reference_energy": 825268.6875 }, { "first_row": 72704, "rows": 1024, "reference_sse": 195.9171600341797, "reference_energy": 833096.25 }, { "first_row": 83968, "rows": 1024, "reference_sse": 225.7914276123047, "reference_energy": 823285.9375 }, { "first_row": 84992, "rows": 1024, "reference_sse": 266.2723388671875, "reference_energy": 388802.125 }, { "first_row": 96256, "rows": 1024, "reference_sse": 211.28375244140625, "reference_energy": 396545.9375 }, { "first_row": 97280, "rows": 1024, "reference_sse": 217.22901916503906, "reference_energy": 835585.9375 } ] }, "best_step": 5, "steps_run": 30, "history": [ { "step": 5, "validation": { "target_rel": 0.004101365709508885, "reference_rel": 0.012358429863239363, "boundary_rel": 0.0845825288851992, "bulk_rel": 0.004094902819067128, "boundary_rows": 24 }, "reasoning_validation": null, "training_probe_rel": 0.4579994575363157, "batch_loss": 0.20938605070114136, "best_step": 5, "overfit_checks": 0, "seconds": 52.11638784408569, "adaptive_schedule": { "best": 0.004101365709508885, "drop_after": 25, "relative_worsening": 0.001, "checks": 3, "patience_updates": 10, "bad_checks": 0, "last_best": 5, "early_drop": false }, "adaptive_stop": false, "quantized_movement": { "codebook_changed": 0, "codebook_values": 1359872, "scales_changed": 48733498, "scale_values": 48955392 } }, { "step": 10, "validation": { "target_rel": 0.004303861889341927, "reference_rel": 0.012418637511646688, "boundary_rel": 0.08699369444106524, "bulk_rel": 0.004297347743661024, "boundary_rows": 24 }, "reasoning_validation": null, "training_probe_rel": 0.47151187172208614, "batch_loss": 0.21316469833254814, "best_step": 5, "overfit_checks": 0, "seconds": 116.97970008850098, "adaptive_schedule": { "best": 0.004101365709508885, "drop_after": 25, "relative_worsening": 0.001, "checks": 3, "patience_updates": 10, "bad_checks": 1, "last_best": 5, "early_drop": false }, "adaptive_stop": false, "quantized_movement": { "codebook_changed": 32904, "codebook_values": 1359872, "scales_changed": 48765228, "scale_values": 48955392 } }, { "step": 15, "validation": { "target_rel": 0.004331628432569988, "reference_rel": 0.012430842634048086, "boundary_rel": 0.08767197261035313, "bulk_rel": 0.004325054668624224, "boundary_rows": 24 }, "reasoning_validation": null, "training_probe_rel": 0.47305746972513324, "batch_loss": 0.2119937352836132, "best_step": 5, "overfit_checks": 0, "seconds": 163.22229027748108, "adaptive_schedule": { "best": 0.004101365709508885, "drop_after": 25, "relative_worsening": 0.001, "checks": 3, "patience_updates": 10, "bad_checks": 2, "last_best": 5, "early_drop": false }, "adaptive_stop": false, "quantized_movement": { "codebook_changed": 32949, "codebook_values": 1359872, "scales_changed": 48789539, "scale_values": 48955392 } }, { "step": 20, "validation": { "target_rel": 0.0043441945894313605, "reference_rel": 0.01243504888771695, "boundary_rel": 0.08796940666377843, "bulk_rel": 0.004337595333739298, "boundary_rows": 24 }, "reasoning_validation": null, "training_probe_rel": 0.47365638787814096, "batch_loss": 0.21270494163036346, "best_step": 5, "overfit_checks": 0, "seconds": 232.2971043586731, "adaptive_schedule": { "best": 0.004101365709508885, "drop_after": 20, "relative_worsening": 0.001, "checks": 3, "patience_updates": 10, "bad_checks": 3, "last_best": 5, "early_drop": true }, "adaptive_stop": false, "quantized_movement": { "codebook_changed": 36520, "codebook_values": 1359872, "scales_changed": 48806517, "scale_values": 48955392 } }, { "step": 25, "validation": { "target_rel": 0.004256532546135758, "reference_rel": 0.012402829315804685, "boundary_rel": 0.08676470668836317, "bulk_rel": 0.004249980218780716, "boundary_rows": 24 }, "reasoning_validation": null, "training_probe_rel": 0.46534569938595405, "batch_loss": 0.21369493752717972, "best_step": 5, "overfit_checks": 0, "seconds": 281.69566798210144, "adaptive_schedule": { "best": 0.004101365709508885, "drop_after": 20, "relative_worsening": 0.001, "checks": 3, "patience_updates": 10, "bad_checks": 3, "last_best": 5, "early_drop": true }, "adaptive_stop": false, "quantized_movement": { "codebook_changed": 16588, "codebook_values": 1359872, "scales_changed": 48808136, "scale_values": 48955392 } }, { "step": 30, "validation": { "target_rel": 0.004287597780785752, "reference_rel": 0.012415740566530201, "boundary_rel": 0.08736656726931337, "bulk_rel": 0.004281002415226635, "boundary_rows": 24 }, "reasoning_validation": null, "training_probe_rel": 0.467414934828306, "batch_loss": 0.19926052168011665, "best_step": 5, "overfit_checks": 0, "seconds": 350.2222332954407, "adaptive_schedule": { "best": 0.004101365709508885, "drop_after": 20, "relative_worsening": 0.001, "checks": 3, "patience_updates": 10, "bad_checks": 3, "last_best": 5, "early_drop": true }, "adaptive_stop": true, "quantized_movement": { "codebook_changed": 24157, "codebook_values": 1359872, "scales_changed": 48809243, "scale_values": 48955392 } } ], "index_reassignments": [ { "method": "expert_parallel_output_gradient_prefix_backtracking_v1", "proposals": 332, "accepted": 280, "changed_groups": 138335, "training_before": 85.35757446289062, "training_after": 84.17408752441406, "acceptance": "proposal and separate training check batches", "step": 10 }, { "method": "expert_parallel_output_gradient_prefix_backtracking_v1", "proposals": 332, "accepted": 266, "changed_groups": 128188, "training_before": 88.69874572753906, "training_after": 87.60824584960938, "acceptance": "proposal and separate training check batches", "step": 20 }, { "method": "expert_parallel_output_gradient_prefix_backtracking_v1", "proposals": 332, "accepted": 275, "changed_groups": 75101, "training_before": 83.35973358154297, "training_after": 82.50680541992188, "acceptance": "proposal and separate training check batches", "step": 30 } ], "coverage_threshold": 256, "checkpoint_selection": "held-out original reference trajectory for both arms", "world_size": 8, "microbatch": 65536, "proposal_communication": "sparse routed rows", "parallelism": "expert-sharded coupled output", "arithmetic": "fp4_planes4_fp16_boundaries_v1", "scope": "text-only, cached prompt partitions; historical audit is development only", "train_rows": 15007754, "validation_rows": 16384, "audit_rows": 16384, "complete": true }, "version": 4, "block_scale_dtype": "float16", "codebook_scope": "expert", "bits": 2.125, "codebook_sizes": [ 256, 256 ], "index_bits": [ 8, 8 ], "words_per_tile": 64, "production_ready": true, "roundtrip": "all expert indices exact; 3 decoded weights/projection exact", "requires_8x8_loader_kernel": true, "production_gate": "sm120 smoke+speed 2026-09-22: coherence PASS; code 128.5 / prose 90.5 / count 140.6 / pangram 138.2 tok/s single-stream (MTP ns=3, tok/step 2.51-3.97); image glm53-arvq-sm120:v4v5-d7ada6d-20260922 @ 950K ctx" }