| { |
| "broader_task_accuracy_validated": false, |
| "evidence_sha256": { |
| "1m-conversation-reuse-v1": "704314a001eb9ceecb80e7d7dafab2cb702365e055d189b269004a3b655065a8", |
| "1m-prefix-reuse-v1": "ad03ae02fef78b1aa911495548fb50f4616b5d9275ae05646027030d3d05ea2e", |
| "1m-v1": "57b638063357aaf36e9b92da4d2ce65d2110bff0a673513ff3d0538b71ed3c3e", |
| "portable_completion_audit": "ab550cd400f30777307f3cae5f5705cb1974a099eafd6cb9ab7bc27f39059eab", |
| "post-1m-generation-v1": "38aa28979314920b87d1455ee29041a1283bc9d4390a6f290abc387528a51d2c", |
| "post-1m-speed-v1": "f2b9c350b72aef3494b15e1d1f7b45b6b3f8e7c9d63bb4b2401ee3756725f694", |
| "source_verification": "9cd9aafd700d5a0e9dc1ceb94f8d9f0f3198192820deb7ca6b65db9b3ad26cb9", |
| "teacher_forced": "74aa83b6b479030ca961a69e717f306f14dcc41e34d1559e679c2d1ad9d6264d" |
| }, |
| "format": "ds41_release_evaluation_summary_v2", |
| "historical_v20": { |
| "evidence_sha256": { |
| "1m-conversation-reuse-v1": "8bf9a68ba10f38ff6ad5a2e6b6afc7aeabc78337a6c08aa74f943fe3489f8681", |
| "1m-prefix-reuse-v1": "336f20b771cf33d88e60c2c7d8050ffc773d88e2806011b57c098e0b83015e3c", |
| "1m-v1": "61f7a1f10c10929e9c76732ec3d073d4ec8ef9eb171aa6f32c1e7045b991eaed", |
| "post-1m-generation-v1": "ea632d06736c3e989dfdcc89735a27b6a009bc7830eef87052bd167f86185e38", |
| "post-1m-speed-v1": "99b44995e37653923220477fdcec84b798acea1b2dca8a8a6aedf0e7ffb1c918" |
| }, |
| "runtime": { |
| "1m-conversation-reuse-v1": { |
| "cache": { |
| "cached_fraction": 0.9997948518305418, |
| "cached_tokens": 1047808, |
| "queried_tokens": 1048023 |
| }, |
| "completion_tokens": 6, |
| "elapsed_seconds": 5.66604625096079, |
| "prompt_tokens": 1048023, |
| "status": "prefix_reuse_pass", |
| "timing": { |
| "decode_tokens": 5, |
| "decode_tokens_per_second": 2.3352113486226043, |
| "server_decode_seconds": 2.1411338219768368, |
| "server_prefill_seconds": 1.538540476001799, |
| "server_time_to_first_token_seconds": 3.4542484283447266 |
| } |
| }, |
| "1m-prefix-reuse-v1": { |
| "cache": { |
| "cached_fraction": 0.9998263342207464, |
| "cached_tokens": 1047808, |
| "queried_tokens": 1047990 |
| }, |
| "completion_tokens": 18, |
| "elapsed_seconds": 10.609135606966447, |
| "prompt_tokens": 1047990, |
| "status": "prefix_reuse_pass", |
| "timing": { |
| "decode_tokens": 17, |
| "decode_tokens_per_second": 2.3254391977644855, |
| "server_decode_seconds": 7.3104469969985075, |
| "server_prefill_seconds": 1.226647932024207, |
| "server_time_to_first_token_seconds": 3.2306277751922607 |
| } |
| }, |
| "1m-v1": { |
| "completion_tokens": 18, |
| "elapsed_seconds": 4331.969199170009, |
| "prompt_tokens": 1047990, |
| "status": "retrieval_pass" |
| }, |
| "post-1m-generation-v1": { |
| "groups": { |
| "diagnostic": { |
| "cases": 2, |
| "exact_matches": 2 |
| }, |
| "heldout_image": { |
| "cases": 24, |
| "exact_matches": 18 |
| }, |
| "text": { |
| "cases": 3, |
| "exact_matches": 3 |
| } |
| }, |
| "original_image_misses_retained": true, |
| "status": "serial_generation_diagnostics_complete" |
| }, |
| "post-1m-speed-v1": { |
| "completion_tokens": 128, |
| "elapsed_seconds": 18.34797103703022, |
| "prompt_tokens": 24, |
| "status": "serial_streaming_speed_measured", |
| "timing": { |
| "decode_tokens": 127, |
| "decode_tokens_per_second": 7.074289903876008, |
| "server_decode_seconds": 17.95233185600955, |
| "server_prefill_seconds": 0.3886793210403994, |
| "server_time_to_first_token_seconds": 0.3920152187347412 |
| } |
| } |
| } |
| }, |
| "portable_v5": { |
| "all_29_replies_identical_to_pre1m": true, |
| "appended_answer_also_present_in_prior_assistant_reply": true, |
| "artifact_content_review_complete": false, |
| "broad_long_context_quality_validated": false, |
| "current_liveness_claimed": false, |
| "mxfp8_aot_sha256": "6abdf60fb353da15d87030427e16a982e7819a6f479563b8acfde81110e78bf4", |
| "publication_approved": false, |
| "runtime_kit_archive_sha256": "8fcabd0558a58229369870fe60f280d01e9c89e9d3194e9c3bfe6cc4e2b1b126", |
| "runtime_kit_manifest_sha256": "bada534aaf8112e734a4afe320f2584a4734e5987d57b421caa1985645d41efc", |
| "same_pair_1m_reuse_and_post1m_regression_passed": true, |
| "separate_cold_vision_variation_resolved": false, |
| "six_session_capacity_validated": false, |
| "synthetic_retrieval_only": true |
| }, |
| "public_repo_id": "coolbho3k/DeepSeek-V4.1-Flash-EXL3-3bpw", |
| "publication_approved": true, |
| "publication_scope": "experimental canonical weights; runtime bundle remains pending", |
| "reference_precision": "original_native_FP4_not_BF16", |
| "runtime": { |
| "1m-conversation-reuse-v1": { |
| "cache": { |
| "cached_fraction": 0.9997948518305418, |
| "cached_tokens": 1047808, |
| "queried_tokens": 1048023 |
| }, |
| "completion_tokens": 6, |
| "elapsed_seconds": 5.315936694969423, |
| "min_available_bytes_by_host": [ |
| 1284665344, |
| 2416865280 |
| ], |
| "prompt_tokens": 1048023, |
| "status": "prefix_reuse_pass", |
| "timing": { |
| "decode_tokens": 5, |
| "decode_tokens_per_second": 2.4060350778472883, |
| "server_decode_seconds": 2.078107690962497, |
| "server_prefill_seconds": 1.40854712104192, |
| "server_time_to_first_token_seconds": 3.1856417655944824 |
| }, |
| "watch_samples": 2 |
| }, |
| "1m-prefix-reuse-v1": { |
| "cache": { |
| "cached_fraction": 0.9998263342207464, |
| "cached_tokens": 1047808, |
| "queried_tokens": 1047990 |
| }, |
| "completion_tokens": 18, |
| "elapsed_seconds": 10.070301063999068, |
| "min_available_bytes_by_host": [ |
| 1229701120, |
| 2506510336 |
| ], |
| "prompt_tokens": 1047990, |
| "status": "prefix_reuse_pass", |
| "timing": { |
| "decode_tokens": 17, |
| "decode_tokens_per_second": 2.4064247723951757, |
| "server_decode_seconds": 7.064421957009472, |
| "server_prefill_seconds": 1.1350557879777625, |
| "server_time_to_first_token_seconds": 2.9549875259399414 |
| }, |
| "watch_samples": 2 |
| }, |
| "1m-v1": { |
| "completion_tokens": 18, |
| "elapsed_seconds": 4277.418559061014, |
| "min_available_bytes_by_host": [ |
| 934481920, |
| 2526097408 |
| ], |
| "prompt_tokens": 1047990, |
| "status": "retrieval_pass", |
| "watch_samples": 411 |
| }, |
| "post-1m-generation-v1": { |
| "groups": { |
| "diagnostic": { |
| "cases": 2, |
| "exact_matches": 2 |
| }, |
| "heldout_image": { |
| "cases": 24, |
| "exact_matches": 18 |
| }, |
| "text": { |
| "cases": 3, |
| "exact_matches": 3 |
| } |
| }, |
| "min_available_bytes_by_host": [ |
| 1168781312, |
| 2507943936 |
| ], |
| "original_image_misses_retained": true, |
| "status": "serial_generation_diagnostics_complete", |
| "watch_samples": 10 |
| }, |
| "post-1m-speed-v1": { |
| "completion_tokens": 128, |
| "elapsed_seconds": 18.073677037027664, |
| "min_available_bytes_by_host": [ |
| 992153600, |
| 2705260544 |
| ], |
| "prompt_tokens": 24, |
| "status": "serial_streaming_speed_measured", |
| "timing": { |
| "decode_tokens": 127, |
| "decode_tokens_per_second": 7.178763194603712, |
| "server_decode_seconds": 17.691069695050828, |
| "server_prefill_seconds": 0.37656140996841714, |
| "server_time_to_first_token_seconds": 0.37935829162597656 |
| }, |
| "watch_samples": 3 |
| } |
| }, |
| "selected_manifest_sha256": "4639e1cf64c5845bb89ae4f9b3648293014bb8c0c32daee2d19f7df0a3f5e86e", |
| "serving_scope": { |
| "clean_external_rebuild_qualified": false, |
| "decode_context_parallel_size": 2, |
| "engrams": "SSD", |
| "gpu_memory_utilization": 0.9, |
| "kv_bytes_per_rank": 1009612800, |
| "kv_dtype": "FP8", |
| "max_model_len": 1048576, |
| "max_num_seqs": 1, |
| "portable_image_deployment_qualified": true, |
| "six_session_support": false, |
| "tensor_parallel_size": 2, |
| "vision": "original_BF16" |
| }, |
| "source_revision": "df42c109f1defefcbfcedbe7d905718a12266e40", |
| "teacher_forced": { |
| "groups": { |
| "image/all_text_targets": { |
| "candidate_mean_nll": 6.373640381638264, |
| "candidate_perplexity": 586.1878970654378, |
| "candidate_target_top1_fraction": 0.45813953488372094, |
| "mean_delta_nll": -0.0569212955755995, |
| "mean_reference_to_candidate_kl": 0.252924389260637, |
| "reference_mean_nll": 6.430561677213864, |
| "reference_perplexity": 620.5223834329012, |
| "reference_target_top1_fraction": 0.46511627906976744, |
| "tokens": 430, |
| "top1_agreement_fraction": 0.8465116279069768 |
| }, |
| "image/answer_only": { |
| "candidate_mean_nll": 4.947521492143288, |
| "candidate_perplexity": 140.8254939251917, |
| "candidate_target_top1_fraction": 0.7023809523809523, |
| "mean_delta_nll": -0.18687153894962297, |
| "mean_reference_to_candidate_kl": 0.025923057464558877, |
| "reference_mean_nll": 5.134393031092911, |
| "reference_perplexity": 169.76124879772908, |
| "reference_target_top1_fraction": 0.7023809523809523, |
| "tokens": 84, |
| "top1_agreement_fraction": 0.9880952380952381 |
| }, |
| "image/answer_with_eos": { |
| "candidate_mean_nll": 3.849337646292081, |
| "candidate_perplexity": 46.96194750784644, |
| "candidate_target_top1_fraction": 0.7685185185185185, |
| "mean_delta_nll": -0.14499815545465125, |
| "mean_reference_to_candidate_kl": 0.020472454994558274, |
| "reference_mean_nll": 3.9943358017467316, |
| "reference_perplexity": 54.289769475886025, |
| "reference_target_top1_fraction": 0.7685185185185185, |
| "tokens": 108, |
| "top1_agreement_fraction": 0.9907407407407407 |
| }, |
| "text/all_text_targets": { |
| "candidate_mean_nll": 1.340283139859622, |
| "candidate_perplexity": 3.820124981905668, |
| "candidate_target_top1_fraction": 0.7060820713238887, |
| "mean_delta_nll": 0.005893201306011715, |
| "mean_reference_to_candidate_kl": 0.044754258236146816, |
| "reference_mean_nll": 1.3343899385536104, |
| "reference_perplexity": 3.7976784223838322, |
| "reference_target_top1_fraction": 0.7071690278456277, |
| "tokens": 81880, |
| "top1_agreement_fraction": 0.9324255007327796 |
| } |
| }, |
| "layers": 40, |
| "records": 64 |
| } |
| } |
|
|
