{ "layer": 10, "cold_expert_ids": [ 0, 2, 3, 4, 5, 6, 8, 10, 11, 12, 13, 15, 16, 17, 18, 19, 23, 24, 25, 26, 28, 29, 31, 32, 35, 36, 37, 38, 40, 42, 43, 46, 49, 50, 51, 52, 53, 54, 55, 56, 57, 59, 60, 61, 62, 63, 64, 65, 66, 68, 69, 70, 71, 73, 74, 75, 76, 77, 79, 80, 81, 83, 85, 90, 93, 94, 96, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 110, 112, 114, 115, 116, 120, 121, 122, 123, 124, 126, 127, 129, 130, 134, 137, 138, 140, 143, 144, 146, 147, 150, 151, 152, 153, 154, 156, 160, 163, 164, 168, 170, 171, 172, 175, 176, 177, 178, 179, 181, 182, 183, 184, 185, 186, 190, 191, 192, 193, 196, 198, 199, 200, 201, 202, 203, 204, 206, 207, 208, 209, 211, 212, 214, 218, 219, 220, 221, 222, 224, 225, 228, 229, 231, 233, 234, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 247, 248, 250, 251, 252, 253, 254, 255 ], "files": { "w13": { "file": "arvq-layer-010-gateup.safetensors", "sha256": "58d0c079753ff41102cd1d7ad43d91e23fa2cb423bfc8baf563d4cd5c1b74fca" }, "w2": { "file": "arvq-layer-010-down.safetensors", "sha256": "69d315727e7cdb1884b89759be39960ac08e688618166c866917ba2a6fb9c6e3" } }, "per_proj": { "w13": { "cb_rel_l2": 0.3405617699797219, "hess_recon_rel_fro": 0.13351171290922292, "diagnostic_metric": "regularized fitting Hessian; mean expert relative error for expert scope", "seconds": 335.3251895904541 }, "w2": { "cb_rel_l2": 0.33818224146001585, "hess_recon_rel_fro": 0.28695327864681747, "diagnostic_metric": "regularized fitting Hessian; mean expert relative error for expert scope", "seconds": 112.27513551712036 } }, "no_rotation": true, "source_identity": { "source": { "repo": "zai-org/GLM-5.3", "revision": "aca966e4e02791568aa6a4ced368624b3d897f42", "meta": { "kind": "block_fp8", "block": [ 128, 128 ], "fmt": "e4m3" }, "cache": "/tmp/glm53-fp8-cold", "legacy_cache_revision_unverified": true }, "capture_sha256": "d1aebdf78d361fc41fca616247621e432d2d4a58e00dd167c82077ae1aa49d66", "cold_ids": [ 0, 2, 3, 4, 5, 6, 8, 10, 11, 12, 13, 15, 16, 17, 18, 19, 23, 24, 25, 26, 28, 29, 31, 32, 35, 36, 37, 38, 40, 42, 43, 46, 49, 50, 51, 52, 53, 54, 55, 56, 57, 59, 60, 61, 62, 63, 64, 65, 66, 68, 69, 70, 71, 73, 74, 75, 76, 77, 79, 80, 81, 83, 85, 90, 93, 94, 96, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 110, 112, 114, 115, 116, 120, 121, 122, 123, 124, 126, 127, 129, 130, 134, 137, 138, 140, 143, 144, 146, 147, 150, 151, 152, 153, 154, 156, 160, 163, 164, 168, 170, 171, 172, 175, 176, 177, 178, 179, 181, 182, 183, 184, 185, 186, 190, 191, 192, 193, 196, 198, 199, 200, 201, 202, 203, 204, 206, 207, 208, 209, 211, 212, 214, 218, 219, 220, 221, 222, 224, 225, 228, 229, 231, 233, 234, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 247, 248, 250, 251, 252, 253, 254, 255 ], "encoder_sha256": "6c20c56fc7a308724ab4faf4174c4e8e32cba5a43c3fccd3f3e6d4da1f0b583e", "settings": { "cb_iters": 8, "sweep_passes": 2, "refine": 1, "col_block": 128, "codebook_scope": "expert" } }, "format": "rvq256_256x8_expert_fp16block", "fit": "sequential same_input-trajectory gradient PV; recipe=full_corpus_sequence_pass_v1; updates=30", "pv_report": { "layer": 10, "target": "same_input", "recipe": "full_corpus_sequence_pass_v1", "block_scale_dtype": "fp16", "batch_tokens": 262144, "sequence_order_sha256": "5e0ef183802009d0483fe37a60f5acd49dd99218130e0c952f097fa473d863bf", "training_sequence_count": 14657, "training_passes": 0.5240171180844249, "training_tokens_seen": 7864320, "adaptive_schedule": { "best": 0.0013590989109528494, "drop_after": 20, "relative_worsening": 0.001, "checks": 3, "patience_updates": 10, "bad_checks": 3, "last_best": 5, "early_drop": true }, "refinement": null, "lr_decay_after": 25, "lr_decay_factor": 0.25, "export_only": false, "training_output_export": "fixed validation/audit outputs only; full training trajectory export not performed", "codebook_lr": 0.048, "scale_lr": 0.032, "quantized_movement": { "codebook_changed": 0, "codebook_values": 1417216, "scales_changed": 50803034, "scale_values": 51019776 }, "initial_validation": { "target_rel": 0.0014691753761509484, "reference_rel": 0.00757221424788843, "boundary_rel": 0.038534116155785836, "bulk_rel": 0.0014680676177891438, "boundary_rows": 24, "windows": [ { "first_row": 8192, "rows": 1024, "reference_sse": 25.011573791503906, "reference_energy": 818861.0625 }, { "first_row": 9216, "rows": 1024, "reference_sse": 24.216327667236328, "reference_energy": 395314.8125 }, { "first_row": 20480, "rows": 1024, "reference_sse": 18.150114059448242, "reference_energy": 816017.5 }, { "first_row": 21504, "rows": 1024, "reference_sse": 28.915172576904297, "reference_energy": 358690.90625 }, { "first_row": 32768, "rows": 1024, "reference_sse": 30.045124053955078, "reference_energy": 818079.9375 }, { "first_row": 33792, "rows": 1024, "reference_sse": 55.44493103027344, "reference_energy": 362598.875 }, { "first_row": 45056, "rows": 1024, "reference_sse": 34.00031280517578, "reference_energy": 826767.625 }, { "first_row": 46080, "rows": 1024, "reference_sse": 37.56602478027344, "reference_energy": 801294.875 }, { "first_row": 57344, "rows": 1024, "reference_sse": 47.160194396972656, "reference_energy": 825061.25 }, { "first_row": 58368, "rows": 1024, "reference_sse": 20.542570114135742, "reference_energy": 828309.3125 }, { "first_row": 69632, "rows": 1024, "reference_sse": 34.233123779296875, "reference_energy": 807070.75 }, { "first_row": 70656, "rows": 1024, "reference_sse": 39.41438674926758, "reference_energy": 393484.0625 }, { "first_row": 81920, "rows": 1024, "reference_sse": 34.145118713378906, "reference_energy": 808544.6875 }, { "first_row": 82944, "rows": 1024, "reference_sse": 48.22911071777344, "reference_energy": 391114.875 }, { "first_row": 94208, "rows": 1024, "reference_sse": 75.21992492675781, "reference_energy": 807968.0 }, { "first_row": 95232, "rows": 1024, "reference_sse": 47.30467987060547, "reference_energy": 398008.8125 } ] }, "final_validation": { "target_rel": 0.0013590989109528494, "reference_rel": 0.007548822557842089, "boundary_rel": 0.03751980130732277, "bulk_rel": 0.0013579634441099313, "boundary_rows": 24, "windows": [ { "first_row": 8192, "rows": 1024, "reference_sse": 24.689598083496094, "reference_energy": 818861.0625 }, { "first_row": 9216, "rows": 1024, "reference_sse": 24.096885681152344, "reference_energy": 395314.8125 }, { "first_row": 20480, "rows": 1024, "reference_sse": 17.838668823242188, "reference_energy": 816017.5 }, { "first_row": 21504, "rows": 1024, "reference_sse": 28.90283966064453, "reference_energy": 358690.90625 }, { "first_row": 32768, "rows": 1024, "reference_sse": 29.78082275390625, "reference_energy": 818079.9375 }, { "first_row": 33792, "rows": 1024, "reference_sse": 55.25505447387695, "reference_energy": 362598.875 }, { "first_row": 45056, "rows": 1024, "reference_sse": 33.86964797973633, "reference_energy": 826767.625 }, { "first_row": 46080, "rows": 1024, "reference_sse": 37.50640869140625, "reference_energy": 801294.875 }, { "first_row": 57344, "rows": 1024, "reference_sse": 47.149131774902344, "reference_energy": 825061.25 }, { "first_row": 58368, "rows": 1024, "reference_sse": 20.35894775390625, "reference_energy": 828309.3125 }, { "first_row": 69632, "rows": 1024, "reference_sse": 34.05101776123047, "reference_energy": 807070.75 }, { "first_row": 70656, "rows": 1024, "reference_sse": 39.05615997314453, "reference_energy": 393484.0625 }, { "first_row": 81920, "rows": 1024, "reference_sse": 33.821144104003906, "reference_energy": 808544.6875 }, { "first_row": 82944, "rows": 1024, "reference_sse": 47.861045837402344, "reference_energy": 391114.875 }, { "first_row": 94208, "rows": 1024, "reference_sse": 75.02511596679688, "reference_energy": 807968.0 }, { "first_row": 95232, "rows": 1024, "reference_sse": 46.63742446899414, "reference_energy": 398008.8125 } ] }, "initial_reasoning_validation": null, "final_reasoning_validation": null, "validation_distribution": "reasoning only", "serialized_validation": { "target_rel": 0.0013590989109528494, "reference_rel": 0.007548822557842089 }, "development_audit": { "target_rel": 0.0021692622873737124, "reference_rel": 0.009149897310529839, "boundary_rel": 0.05932958864768074, "bulk_rel": 0.0021685352613771717, "boundary_rows": 10, "windows": [ { "first_row": 10240, "rows": 1024, "reference_sse": 127.5205078125, "reference_energy": 784687.0 }, { "first_row": 11264, "rows": 1024, "reference_sse": 48.958980560302734, "reference_energy": 817930.875 }, { "first_row": 22528, "rows": 1024, "reference_sse": 77.83999633789062, "reference_energy": 395126.15625 }, { "first_row": 23552, "rows": 1024, "reference_sse": 45.68903350830078, "reference_energy": 389303.375 }, { "first_row": 34816, "rows": 1024, "reference_sse": 61.118412017822266, "reference_energy": 813355.8125 }, { "first_row": 35840, "rows": 1024, "reference_sse": 45.03987121582031, "reference_energy": 819497.0625 }, { "first_row": 47104, "rows": 1024, "reference_sse": 45.806915283203125, "reference_energy": 370809.5 }, { "first_row": 48128, "rows": 1024, "reference_sse": 36.92018127441406, "reference_energy": 392154.9375 }, { "first_row": 59392, "rows": 1024, "reference_sse": 78.35218048095703, "reference_energy": 793577.0625 }, { "first_row": 60416, "rows": 1024, "reference_sse": 46.554595947265625, "reference_energy": 802145.4375 }, { "first_row": 71680, "rows": 1024, "reference_sse": 27.82541275024414, "reference_energy": 809944.9375 }, { "first_row": 72704, "rows": 1024, "reference_sse": 33.662200927734375, "reference_energy": 818017.75 }, { "first_row": 83968, "rows": 1024, "reference_sse": 69.15956115722656, "reference_energy": 809467.125 }, { "first_row": 84992, "rows": 1024, "reference_sse": 49.998619079589844, "reference_energy": 379213.3125 }, { "first_row": 96256, "rows": 1024, "reference_sse": 46.072628021240234, "reference_energy": 387386.625 }, { "first_row": 97280, "rows": 1024, "reference_sse": 30.550865173339844, "reference_energy": 821868.25 } ] }, "initial_development_audit": { "target_rel": 0.0023164123170559994, "reference_rel": 0.009214108550176165, "boundary_rel": 0.06188832313734172, "bulk_rel": 0.002315671569128643, "boundary_rows": 10, "windows": [ { "first_row": 10240, "rows": 1024, "reference_sse": 127.93661499023438, "reference_energy": 784687.0 }, { "first_row": 11264, "rows": 1024, "reference_sse": 50.33224105834961, "reference_energy": 817930.875 }, { "first_row": 22528, "rows": 1024, "reference_sse": 78.21121215820312, "reference_energy": 395126.15625 }, { "first_row": 23552, "rows": 1024, "reference_sse": 46.34546661376953, "reference_energy": 389303.375 }, { "first_row": 34816, "rows": 1024, "reference_sse": 62.206298828125, "reference_energy": 813355.8125 }, { "first_row": 35840, "rows": 1024, "reference_sse": 45.816673278808594, "reference_energy": 819497.0625 }, { "first_row": 47104, "rows": 1024, "reference_sse": 46.26963806152344, "reference_energy": 370809.5 }, { "first_row": 48128, "rows": 1024, "reference_sse": 37.560394287109375, "reference_energy": 392154.9375 }, { "first_row": 59392, "rows": 1024, "reference_sse": 79.11900329589844, "reference_energy": 793577.0625 }, { "first_row": 60416, "rows": 1024, "reference_sse": 47.35142517089844, "reference_energy": 802145.4375 }, { "first_row": 71680, "rows": 1024, "reference_sse": 28.685527801513672, "reference_energy": 809944.9375 }, { "first_row": 72704, "rows": 1024, "reference_sse": 34.45107650756836, "reference_energy": 818017.75 }, { "first_row": 83968, "rows": 1024, "reference_sse": 69.89288330078125, "reference_energy": 809467.125 }, { "first_row": 84992, "rows": 1024, "reference_sse": 50.83473205566406, "reference_energy": 379213.3125 }, { "first_row": 96256, "rows": 1024, "reference_sse": 46.66944885253906, "reference_energy": 387386.625 }, { "first_row": 97280, "rows": 1024, "reference_sse": 31.65604019165039, "reference_energy": 821868.25 } ] }, "best_step": 5, "steps_run": 30, "history": [ { "step": 5, "validation": { "target_rel": 0.0013590989109528494, "reference_rel": 0.007548822557842089, "boundary_rel": 0.03751980130732277, "bulk_rel": 0.0013579634441099313, "boundary_rows": 24 }, "reasoning_validation": null, "training_probe_rel": 0.3762629768712852, "batch_loss": 0.14867597818374634, "best_step": 5, "overfit_checks": 0, "seconds": 52.50445556640625, "adaptive_schedule": { "best": 0.0013590989109528494, "drop_after": 25, "relative_worsening": 0.001, "checks": 3, "patience_updates": 10, "bad_checks": 0, "last_best": 5, "early_drop": false }, "adaptive_stop": false, "quantized_movement": { "codebook_changed": 0, "codebook_values": 1417216, "scales_changed": 50803034, "scale_values": 51019776 } }, { "step": 10, "validation": { "target_rel": 0.0014300483157651677, "reference_rel": 0.007562067924089202, "boundary_rel": 0.0390911740123365, "bulk_rel": 0.00142887693295051, "boundary_rows": 24 }, "reasoning_validation": null, "training_probe_rel": 0.39358266591663654, "batch_loss": 0.1501602940261364, "best_step": 5, "overfit_checks": 0, "seconds": 123.94567227363586, "adaptive_schedule": { "best": 0.0013590989109528494, "drop_after": 25, "relative_worsening": 0.001, "checks": 3, "patience_updates": 10, "bad_checks": 1, "last_best": 5, "early_drop": false }, "adaptive_stop": false, "quantized_movement": { "codebook_changed": 34490, "codebook_values": 1417216, "scales_changed": 50819149, "scale_values": 51019776 } }, { "step": 15, "validation": { "target_rel": 0.0014328196562852125, "reference_rel": 0.007559611099071919, "boundary_rel": 0.03902326466804658, "bulk_rel": 0.0014316546323392958, "boundary_rows": 24 }, "reasoning_validation": null, "training_probe_rel": 0.3950021383790322, "batch_loss": 0.15294718742370605, "best_step": 5, "overfit_checks": 0, "seconds": 176.7134509086609, "adaptive_schedule": { "best": 0.0013590989109528494, "drop_after": 25, "relative_worsening": 0.001, "checks": 3, "patience_updates": 10, "bad_checks": 2, "last_best": 5, "early_drop": false }, "adaptive_stop": false, "quantized_movement": { "codebook_changed": 34055, "codebook_values": 1417216, "scales_changed": 50841099, "scale_values": 51019776 } }, { "step": 20, "validation": { "target_rel": 0.0014278821651175341, "reference_rel": 0.0075594753554706455, "boundary_rel": 0.03906809624635099, "bulk_rel": 0.0014267103821219754, "boundary_rows": 24 }, "reasoning_validation": null, "training_probe_rel": 0.3923422959770039, "batch_loss": 0.15226780995726585, "best_step": 5, "overfit_checks": 0, "seconds": 252.63794350624084, "adaptive_schedule": { "best": 0.0013590989109528494, "drop_after": 20, "relative_worsening": 0.001, "checks": 3, "patience_updates": 10, "bad_checks": 3, "last_best": 5, "early_drop": true }, "adaptive_stop": false, "quantized_movement": { "codebook_changed": 37143, "codebook_values": 1417216, "scales_changed": 50860781, "scale_values": 51019776 } }, { "step": 25, "validation": { "target_rel": 0.001385242950379184, "reference_rel": 0.007552490607965308, "boundary_rel": 0.03841918739769557, "bulk_rel": 0.001384074830625623, "boundary_rows": 24 }, "reasoning_validation": null, "training_probe_rel": 0.3817737838577274, "batch_loss": 0.14436077326536179, "best_step": 5, "overfit_checks": 0, "seconds": 305.20060777664185, "adaptive_schedule": { "best": 0.0013590989109528494, "drop_after": 20, "relative_worsening": 0.001, "checks": 3, "patience_updates": 10, "bad_checks": 3, "last_best": 5, "early_drop": true }, "adaptive_stop": false, "quantized_movement": { "codebook_changed": 18399, "codebook_values": 1417216, "scales_changed": 50862629, "scale_values": 51019776 } }, { "step": 30, "validation": { "target_rel": 0.0013948087983861163, "reference_rel": 0.007553742631912449, "boundary_rel": 0.03850594950703197, "bulk_rel": 0.0013936434569979013, "boundary_rows": 24 }, "reasoning_validation": null, "training_probe_rel": 0.3831294769793443, "batch_loss": 0.1382920742034912, "best_step": 5, "overfit_checks": 0, "seconds": 379.96928453445435, "adaptive_schedule": { "best": 0.0013590989109528494, "drop_after": 20, "relative_worsening": 0.001, "checks": 3, "patience_updates": 10, "bad_checks": 3, "last_best": 5, "early_drop": true }, "adaptive_stop": true, "quantized_movement": { "codebook_changed": 25267, "codebook_values": 1417216, "scales_changed": 50865210, "scale_values": 51019776 } } ], "index_reassignments": [ { "method": "expert_parallel_output_gradient_prefix_backtracking_v1", "proposals": 346, "accepted": 276, "changed_groups": 147877, "training_before": 12.368152618408203, "training_after": 12.184597969055176, "acceptance": "proposal and separate training check batches", "step": 10 }, { "method": "expert_parallel_output_gradient_prefix_backtracking_v1", "proposals": 346, "accepted": 272, "changed_groups": 131161, "training_before": 12.86461353302002, "training_after": 12.62559700012207, "acceptance": "proposal and separate training check batches", "step": 20 }, { "method": "expert_parallel_output_gradient_prefix_backtracking_v1", "proposals": 346, "accepted": 258, "changed_groups": 112347, "training_before": 11.797676086425781, "training_after": 11.688483238220215, "acceptance": "proposal and separate training check batches", "step": 30 } ], "coverage_threshold": 256, "checkpoint_selection": "held-out original reference trajectory for both arms", "world_size": 8, "microbatch": 65536, "proposal_communication": "sparse routed rows", "parallelism": "expert-sharded coupled output", "arithmetic": "fp4_planes4_fp16_boundaries_v1", "scope": "text-only, cached prompt partitions; historical audit is development only", "train_rows": 15007754, "validation_rows": 16384, "audit_rows": 16384, "complete": true }, "version": 4, "block_scale_dtype": "float16", "codebook_scope": "expert", "bits": 2.125, "codebook_sizes": [ 256, 256 ], "index_bits": [ 8, 8 ], "words_per_tile": 64, "production_ready": true, "roundtrip": "all expert indices exact; 3 decoded weights/projection exact", "requires_8x8_loader_kernel": true, "production_gate": "sm120 smoke+speed 2026-09-22: coherence PASS; code 128.5 / prose 90.5 / count 140.6 / pangram 138.2 tok/s single-stream (MTP ns=3, tok/step 2.51-3.97); image glm53-arvq-sm120:v4v5-d7ada6d-20260922 @ 950K ctx" }