{ "layer": 37, "cold_expert_ids": [ 1, 2, 3, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 23, 24, 26, 27, 29, 32, 33, 34, 35, 36, 37, 38, 39, 41, 42, 43, 44, 45, 47, 48, 51, 53, 54, 55, 57, 58, 61, 62, 63, 65, 67, 68, 69, 70, 73, 74, 75, 76, 77, 78, 82, 83, 84, 85, 86, 87, 88, 89, 90, 92, 94, 97, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 114, 116, 118, 119, 121, 123, 124, 125, 126, 128, 129, 130, 132, 134, 135, 136, 137, 138, 139, 140, 141, 142, 145, 146, 147, 148, 149, 152, 153, 155, 156, 157, 158, 161, 162, 163, 166, 167, 168, 170, 171, 172, 174, 176, 177, 178, 179, 180, 183, 185, 187, 191, 192, 193, 194, 195, 196, 197, 199, 200, 201, 202, 203, 204, 205, 206, 208, 210, 211, 212, 214, 215, 217, 218, 219, 221, 223, 224, 225, 228, 229, 231, 232, 235, 237, 239, 241, 242, 244, 245, 246, 247, 248, 249, 250, 251, 253, 254, 255 ], "files": { "w13": { "file": "arvq-layer-037-gateup.safetensors", "sha256": "03d7c9585b1f3473fe8cef03b7f09ccdb0c38dd19d40fd7b054b7a5ab9c0e7ab" }, "w2": { "file": "arvq-layer-037-down.safetensors", "sha256": "67959a27b68b08618787a369544fb29a27f680f77f292555558fd1e3f201945f" } }, "per_proj": { "w13": { "cb_rel_l2": 0.3402764064546334, "hess_recon_rel_fro": 0.1556607101173991, "diagnostic_metric": "regularized fitting Hessian; mean expert relative error for expert scope", "seconds": 326.70758748054504 }, "w2": { "cb_rel_l2": 0.34041954120792445, "hess_recon_rel_fro": 0.2980905843760792, "diagnostic_metric": "regularized fitting Hessian; mean expert relative error for expert scope", "seconds": 108.77585124969482 } }, "no_rotation": true, "source_identity": { "source": { "repo": "zai-org/GLM-5.3", "revision": "aca966e4e02791568aa6a4ced368624b3d897f42", "meta": { "kind": "block_fp8", "block": [ 128, 128 ], "fmt": "e4m3" }, "cache": "/tmp/glm53-fp8-cold", "legacy_cache_revision_unverified": true }, "capture_sha256": "301803bbcb3af2c2da66556972905b457280df05b207d277b14f0e71c9ae9fae", "cold_ids": [ 1, 2, 3, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 23, 24, 26, 27, 29, 32, 33, 34, 35, 36, 37, 38, 39, 41, 42, 43, 44, 45, 47, 48, 51, 53, 54, 55, 57, 58, 61, 62, 63, 65, 67, 68, 69, 70, 73, 74, 75, 76, 77, 78, 82, 83, 84, 85, 86, 87, 88, 89, 90, 92, 94, 97, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 114, 116, 118, 119, 121, 123, 124, 125, 126, 128, 129, 130, 132, 134, 135, 136, 137, 138, 139, 140, 141, 142, 145, 146, 147, 148, 149, 152, 153, 155, 156, 157, 158, 161, 162, 163, 166, 167, 168, 170, 171, 172, 174, 176, 177, 178, 179, 180, 183, 185, 187, 191, 192, 193, 194, 195, 196, 197, 199, 200, 201, 202, 203, 204, 205, 206, 208, 210, 211, 212, 214, 215, 217, 218, 219, 221, 223, 224, 225, 228, 229, 231, 232, 235, 237, 239, 241, 242, 244, 245, 246, 247, 248, 249, 250, 251, 253, 254, 255 ], "encoder_sha256": "6c20c56fc7a308724ab4faf4174c4e8e32cba5a43c3fccd3f3e6d4da1f0b583e", "settings": { "cb_iters": 8, "sweep_passes": 2, "refine": 1, "col_block": 128, "codebook_scope": "expert" } }, "format": "rvq256_256x8_expert_fp16block", "fit": "sequential same_input-trajectory gradient PV; recipe=full_corpus_sequence_pass_v1; updates=30", "pv_report": { "layer": 37, "target": "same_input", "recipe": "full_corpus_sequence_pass_v1", "block_scale_dtype": "fp16", "batch_tokens": 262144, "sequence_order_sha256": "5e0ef183802009d0483fe37a60f5acd49dd99218130e0c952f097fa473d863bf", "training_sequence_count": 14657, "training_passes": 0.5240171180844249, "training_tokens_seen": 7864320, "adaptive_schedule": { "best": 0.035430789227037796, "drop_after": 20, "relative_worsening": 0.001, "checks": 3, "patience_updates": 10, "bad_checks": 3, "last_best": 5, "early_drop": true }, "refinement": null, "lr_decay_after": 25, "lr_decay_factor": 0.25, "export_only": false, "training_output_export": "fixed validation/audit outputs only; full training trajectory export not performed", "codebook_lr": 0.048, "scale_lr": 0.032, "quantized_movement": { "codebook_changed": 0, "codebook_values": 1507328, "scales_changed": 54010475, "scale_values": 54263808 }, "initial_validation": { "target_rel": 0.03638600899968513, "reference_rel": 0.17823004132796647, "boundary_rel": 0.10124323495332303, "bulk_rel": 0.03636844857731987, "boundary_rows": 24, "windows": [ { "first_row": 8192, "rows": 1024, "reference_sse": 35137.57421875, "reference_energy": 3320458.5 }, { "first_row": 9216, "rows": 1024, "reference_sse": 42633.6328125, "reference_energy": 1843775.75 }, { "first_row": 20480, "rows": 1024, "reference_sse": 42635.8125, "reference_energy": 3305440.25 }, { "first_row": 21504, "rows": 1024, "reference_sse": 45802.203125, "reference_energy": 1659533.0 }, { "first_row": 32768, "rows": 1024, "reference_sse": 220005.5625, "reference_energy": 3278694.5 }, { "first_row": 33792, "rows": 1024, "reference_sse": 159158.359375, "reference_energy": 1711627.0 }, { "first_row": 45056, "rows": 1024, "reference_sse": 264612.28125, "reference_energy": 3321431.25 }, { "first_row": 46080, "rows": 1024, "reference_sse": 43517.66015625, "reference_energy": 3240806.75 }, { "first_row": 57344, "rows": 1024, "reference_sse": 253359.390625, "reference_energy": 3353167.5 }, { "first_row": 58368, "rows": 1024, "reference_sse": 33247.46484375, "reference_energy": 3341494.0 }, { "first_row": 69632, "rows": 1024, "reference_sse": 40167.7578125, "reference_energy": 3216909.5 }, { "first_row": 70656, "rows": 1024, "reference_sse": 40524.2578125, "reference_energy": 1782996.25 }, { "first_row": 81920, "rows": 1024, "reference_sse": 45360.5390625, "reference_energy": 3297010.0 }, { "first_row": 82944, "rows": 1024, "reference_sse": 44772.4921875, "reference_energy": 1812945.0 }, { "first_row": 94208, "rows": 1024, "reference_sse": 42648.7421875, "reference_energy": 3210597.25 }, { "first_row": 95232, "rows": 1024, "reference_sse": 28247.5390625, "reference_energy": 1803508.5 } ] }, "final_validation": { "target_rel": 0.035430789227037796, "reference_rel": 0.17826474279393106, "boundary_rel": 0.0990677845146195, "bulk_rel": 0.035413497301755595, "boundary_rows": 24, "windows": [ { "first_row": 8192, "rows": 1024, "reference_sse": 35089.94140625, "reference_energy": 3320458.5 }, { "first_row": 9216, "rows": 1024, "reference_sse": 42676.2734375, "reference_energy": 1843775.75 }, { "first_row": 20480, "rows": 1024, "reference_sse": 42627.18359375, "reference_energy": 3305440.25 }, { "first_row": 21504, "rows": 1024, "reference_sse": 45848.0546875, "reference_energy": 1659533.0 }, { "first_row": 32768, "rows": 1024, "reference_sse": 219938.3125, "reference_energy": 3278694.5 }, { "first_row": 33792, "rows": 1024, "reference_sse": 159318.109375, "reference_energy": 1711627.0 }, { "first_row": 45056, "rows": 1024, "reference_sse": 264638.8125, "reference_energy": 3321431.25 }, { "first_row": 46080, "rows": 1024, "reference_sse": 43503.98046875, "reference_energy": 3240806.75 }, { "first_row": 57344, "rows": 1024, "reference_sse": 253354.53125, "reference_energy": 3353167.5 }, { "first_row": 58368, "rows": 1024, "reference_sse": 33295.421875, "reference_energy": 3341494.0 }, { "first_row": 69632, "rows": 1024, "reference_sse": 40245.65234375, "reference_energy": 3216909.5 }, { "first_row": 70656, "rows": 1024, "reference_sse": 40619.1796875, "reference_energy": 1782996.25 }, { "first_row": 81920, "rows": 1024, "reference_sse": 45421.39453125, "reference_energy": 3297010.0 }, { "first_row": 82944, "rows": 1024, "reference_sse": 44891.0703125, "reference_energy": 1812945.0 }, { "first_row": 94208, "rows": 1024, "reference_sse": 42730.5078125, "reference_energy": 3210597.25 }, { "first_row": 95232, "rows": 1024, "reference_sse": 28170.982421875, "reference_energy": 1803508.5 } ] }, "initial_reasoning_validation": null, "final_reasoning_validation": null, "validation_distribution": "reasoning only", "serialized_validation": { "target_rel": 0.035430789227037796, "reference_rel": 0.17826474279393106 }, "development_audit": { "target_rel": 0.045509357369676456, "reference_rel": 0.1948498272178212, "boundary_rel": 0.07109039092744861, "bulk_rel": 0.045507685106836004, "boundary_rows": 10, "windows": [ { "first_row": 10240, "rows": 1024, "reference_sse": 75182.3203125, "reference_energy": 3193274.5 }, { "first_row": 11264, "rows": 1024, "reference_sse": 97508.15625, "reference_energy": 3317568.0 }, { "first_row": 22528, "rows": 1024, "reference_sse": 207918.9375, "reference_energy": 1884921.25 }, { "first_row": 23552, "rows": 1024, "reference_sse": 93546.0859375, "reference_energy": 1767864.75 }, { "first_row": 34816, "rows": 1024, "reference_sse": 69537.34375, "reference_energy": 3427986.0 }, { "first_row": 35840, "rows": 1024, "reference_sse": 68986.140625, "reference_energy": 3419562.0 }, { "first_row": 47104, "rows": 1024, "reference_sse": 71392.5625, "reference_energy": 1795405.5 }, { "first_row": 48128, "rows": 1024, "reference_sse": 194320.3125, "reference_energy": 1805939.875 }, { "first_row": 59392, "rows": 1024, "reference_sse": 79444.125, "reference_energy": 3302534.5 }, { "first_row": 60416, "rows": 1024, "reference_sse": 90815.125, "reference_energy": 3199670.0 }, { "first_row": 71680, "rows": 1024, "reference_sse": 81792.359375, "reference_energy": 3355935.5 }, { "first_row": 72704, "rows": 1024, "reference_sse": 113013.1171875, "reference_energy": 3249233.5 }, { "first_row": 83968, "rows": 1024, "reference_sse": 91647.0625, "reference_energy": 3249174.0 }, { "first_row": 84992, "rows": 1024, "reference_sse": 128234.5, "reference_energy": 1697095.5 }, { "first_row": 96256, "rows": 1024, "reference_sse": 103288.109375, "reference_energy": 1790603.875 }, { "first_row": 97280, "rows": 1024, "reference_sse": 94954.078125, "reference_energy": 3307663.0 } ] }, "initial_development_audit": { "target_rel": 0.046305665997648224, "reference_rel": 0.19487091091451192, "boundary_rel": 0.0723511821311757, "bulk_rel": 0.046303963290075564, "boundary_rows": 10, "windows": [ { "first_row": 10240, "rows": 1024, "reference_sse": 75198.5625, "reference_energy": 3193274.5 }, { "first_row": 11264, "rows": 1024, "reference_sse": 97442.171875, "reference_energy": 3317568.0 }, { "first_row": 22528, "rows": 1024, "reference_sse": 207988.546875, "reference_energy": 1884921.25 }, { "first_row": 23552, "rows": 1024, "reference_sse": 93582.046875, "reference_energy": 1767864.75 }, { "first_row": 34816, "rows": 1024, "reference_sse": 69757.234375, "reference_energy": 3427986.0 }, { "first_row": 35840, "rows": 1024, "reference_sse": 69066.28125, "reference_energy": 3419562.0 }, { "first_row": 47104, "rows": 1024, "reference_sse": 71672.6796875, "reference_energy": 1795405.5 }, { "first_row": 48128, "rows": 1024, "reference_sse": 194239.078125, "reference_energy": 1805939.875 }, { "first_row": 59392, "rows": 1024, "reference_sse": 79530.5625, "reference_energy": 3302534.5 }, { "first_row": 60416, "rows": 1024, "reference_sse": 90734.734375, "reference_energy": 3199670.0 }, { "first_row": 71680, "rows": 1024, "reference_sse": 81931.75, "reference_energy": 3355935.5 }, { "first_row": 72704, "rows": 1024, "reference_sse": 113024.171875, "reference_energy": 3249233.5 }, { "first_row": 83968, "rows": 1024, "reference_sse": 91577.21875, "reference_energy": 3249174.0 }, { "first_row": 84992, "rows": 1024, "reference_sse": 128104.8046875, "reference_energy": 1697095.5 }, { "first_row": 96256, "rows": 1024, "reference_sse": 103302.296875, "reference_energy": 1790603.875 }, { "first_row": 97280, "rows": 1024, "reference_sse": 94787.796875, "reference_energy": 3307663.0 } ] }, "best_step": 5, "steps_run": 30, "history": [ { "step": 5, "validation": { "target_rel": 0.035430789227037796, "reference_rel": 0.17826474279393106, "boundary_rel": 0.0990677845146195, "bulk_rel": 0.035413497301755595, "boundary_rows": 24 }, "reasoning_validation": null, "training_probe_rel": 0.6033699401631855, "batch_loss": 0.3711230754852295, "best_step": 5, "overfit_checks": 0, "seconds": 50.20509743690491, "adaptive_schedule": { "best": 0.035430789227037796, "drop_after": 25, "relative_worsening": 0.001, "checks": 3, "patience_updates": 10, "bad_checks": 0, "last_best": 5, "early_drop": false }, "adaptive_stop": false, "quantized_movement": { "codebook_changed": 0, "codebook_values": 1507328, "scales_changed": 54010475, "scale_values": 54263808 } }, { "step": 10, "validation": { "target_rel": 0.03619564868686447, "reference_rel": 0.17819636515064755, "boundary_rel": 0.10062735507144575, "bulk_rel": 0.036178214778716994, "boundary_rows": 24 }, "reasoning_validation": null, "training_probe_rel": 0.6154018346610161, "batch_loss": 0.4006205275654793, "best_step": 5, "overfit_checks": 0, "seconds": 122.71369981765747, "adaptive_schedule": { "best": 0.035430789227037796, "drop_after": 25, "relative_worsening": 0.001, "checks": 3, "patience_updates": 10, "bad_checks": 1, "last_best": 5, "early_drop": false }, "adaptive_stop": false, "quantized_movement": { "codebook_changed": 61610, "codebook_values": 1507328, "scales_changed": 54061165, "scale_values": 54263808 } }, { "step": 15, "validation": { "target_rel": 0.036094766442899044, "reference_rel": 0.17829567656381615, "boundary_rel": 0.10063360503731715, "bulk_rel": 0.03607726666285808, "boundary_rows": 24 }, "reasoning_validation": null, "training_probe_rel": 0.615315198740628, "batch_loss": 0.37074875831604004, "best_step": 5, "overfit_checks": 0, "seconds": 170.382803440094, "adaptive_schedule": { "best": 0.035430789227037796, "drop_after": 25, "relative_worsening": 0.001, "checks": 3, "patience_updates": 10, "bad_checks": 2, "last_best": 5, "early_drop": false }, "adaptive_stop": false, "quantized_movement": { "codebook_changed": 51090, "codebook_values": 1507328, "scales_changed": 54091011, "scale_values": 54263808 } }, { "step": 20, "validation": { "target_rel": 0.03600335932378524, "reference_rel": 0.17828485097818095, "boundary_rel": 0.10027343284427055, "bulk_rel": 0.03598594606919889, "boundary_rows": 24 }, "reasoning_validation": null, "training_probe_rel": 0.6158801987724516, "batch_loss": 0.37650736421346664, "best_step": 5, "overfit_checks": 0, "seconds": 241.6281168460846, "adaptive_schedule": { "best": 0.035430789227037796, "drop_after": 20, "relative_worsening": 0.001, "checks": 3, "patience_updates": 10, "bad_checks": 3, "last_best": 5, "early_drop": true }, "adaptive_stop": false, "quantized_movement": { "codebook_changed": 50916, "codebook_values": 1507328, "scales_changed": 54106965, "scale_values": 54263808 } }, { "step": 25, "validation": { "target_rel": 0.035631185110012245, "reference_rel": 0.1781897039874813, "boundary_rel": 0.09919181315665043, "bulk_rel": 0.03561396966842379, "boundary_rows": 24 }, "reasoning_validation": null, "training_probe_rel": 0.6098113032737027, "batch_loss": 0.37748441845178604, "best_step": 5, "overfit_checks": 0, "seconds": 291.77955889701843, "adaptive_schedule": { "best": 0.035430789227037796, "drop_after": 20, "relative_worsening": 0.001, "checks": 3, "patience_updates": 10, "bad_checks": 3, "last_best": 5, "early_drop": true }, "adaptive_stop": false, "quantized_movement": { "codebook_changed": 26594, "codebook_values": 1507328, "scales_changed": 54108367, "scale_values": 54263808 } }, { "step": 30, "validation": { "target_rel": 0.035747399777122636, "reference_rel": 0.17825168628273177, "boundary_rel": 0.09923035581395905, "bulk_rel": 0.03573024186144143, "boundary_rows": 24 }, "reasoning_validation": null, "training_probe_rel": 0.6115424098203788, "batch_loss": 0.35952814668416977, "best_step": 5, "overfit_checks": 0, "seconds": 363.33829712867737, "adaptive_schedule": { "best": 0.035430789227037796, "drop_after": 20, "relative_worsening": 0.001, "checks": 3, "patience_updates": 10, "bad_checks": 3, "last_best": 5, "early_drop": true }, "adaptive_stop": true, "quantized_movement": { "codebook_changed": 37998, "codebook_values": 1507328, "scales_changed": 54108512, "scale_values": 54263808 } } ], "index_reassignments": [ { "method": "expert_parallel_output_gradient_prefix_backtracking_v1", "proposals": 362, "accepted": 326, "changed_groups": 530047, "training_before": 23899.68359375, "training_after": 23495.0703125, "acceptance": "proposal and separate training check batches", "step": 10 }, { "method": "expert_parallel_output_gradient_prefix_backtracking_v1", "proposals": 360, "accepted": 305, "changed_groups": 429387, "training_before": 23100.31640625, "training_after": 22729.69921875, "acceptance": "proposal and separate training check batches", "step": 20 }, { "method": "expert_parallel_output_gradient_prefix_backtracking_v1", "proposals": 356, "accepted": 312, "changed_groups": 326218, "training_before": 21741.9375, "training_after": 21466.587890625, "acceptance": "proposal and separate training check batches", "step": 30 } ], "coverage_threshold": 256, "checkpoint_selection": "held-out original reference trajectory for both arms", "world_size": 8, "microbatch": 65536, "proposal_communication": "sparse routed rows", "parallelism": "expert-sharded coupled output", "arithmetic": "fp4_planes4_fp16_boundaries_v1", "scope": "text-only, cached prompt partitions; historical audit is development only", "train_rows": 15007754, "validation_rows": 16384, "audit_rows": 16384, "complete": true }, "version": 4, "block_scale_dtype": "float16", "codebook_scope": "expert", "bits": 2.125, "codebook_sizes": [ 256, 256 ], "index_bits": [ 8, 8 ], "words_per_tile": 64, "production_ready": true, "roundtrip": "all expert indices exact; 3 decoded weights/projection exact", "requires_8x8_loader_kernel": true, "production_gate": "sm120 smoke+speed 2026-09-22: coherence PASS; code 128.5 / prose 90.5 / count 140.6 / pangram 138.2 tok/s single-stream (MTP ns=3, tok/step 2.51-3.97); image glm53-arvq-sm120:v4v5-d7ada6d-20260922 @ 950K ctx" }