| { |
| "schemaVersion": 1, |
| "model": { |
| "repo": "henry1477/translategemma-27b-it-NVFP4A16", |
| "baseRepo": "google/translategemma-27b-it", |
| "baseRevision": "7d10f0b72f89a2d0f268cea30727d8b77c0d25c2", |
| "scheme": "NVFP4A16", |
| "architecture": "Gemma3ForCausalLM", |
| "checkpointSha256": "a636a4d1b71c6e0c7d8450e0d70c2ed494fa74594b4a8652c0da2f24b7f92f5c", |
| "parametersBefore": 27432406640, |
| "parametersAfter": 27009346304 |
| }, |
| "calibration": { |
| "prompts": 512, |
| "cues": 10240, |
| "sourceLanguages": [ |
| "ar", |
| "bg", |
| "ca", |
| "cs", |
| "da", |
| "de", |
| "el", |
| "es", |
| "fi", |
| "fr", |
| "he", |
| "hi", |
| "hu", |
| "id", |
| "it", |
| "ja", |
| "ko", |
| "ms", |
| "nl", |
| "no", |
| "pl", |
| "pt", |
| "ro", |
| "ru", |
| "sv", |
| "th", |
| "tr", |
| "uk", |
| "vi", |
| "zh" |
| ], |
| "targetLanguages": [ |
| "en", |
| "es-MX" |
| ], |
| "sha256": "d5e42cebbf1e5c9f05e04c84751bac3076ca8886c98cad42442115075625159c", |
| "maxSequenceLength": 2048, |
| "rtnFallbackModules": 62, |
| "rtnFallbackPattern": "model.layers.{0..61}.mlp.down_proj" |
| }, |
| "productionProfile": { |
| "maxModelLen": 4096, |
| "constrainedJson": true, |
| "gpuMemoryUtilization": 0.78, |
| "maxNumSeqs": 4, |
| "waveConcurrency": 2, |
| "kvAllocatorHeadroom": true |
| }, |
| "baseline": { |
| "repo": "henry1477/translategemma-12b-it-NVFP4A16", |
| "revision": "a4afd9372aaad181f97ba2ba057524ea95d281e7", |
| "succeeded": true, |
| "structuralPass": true, |
| "metrics": { |
| "engine": "vllm", |
| "label": "12b-real-corpus-production-recovery", |
| "model": "/models/huggingface/models--henry1477--translategemma-12b-it-NVFP4A16/snapshots/a4afd9372aaad181f97ba2ba057524ea95d281e7", |
| "timestamp": "2026-08-28T04:51:20Z", |
| "load_seconds": 97.9, |
| "generate_seconds": 20.5, |
| "first_token_seconds": null, |
| "output_tokens": 10146, |
| "generation_requests": 18, |
| "recovery_splits": 2, |
| "output_tokens_per_second": 494.1, |
| "scenes_per_minute": 40.91, |
| "peak_vram_mib": 30456, |
| "max_num_seqs": 16, |
| "max_model_len": 4096, |
| "constrained_json": true, |
| "clock_drift_pct": 1395.9, |
| "scenes": 14, |
| "validated": 14, |
| "validation_rate": 100.0, |
| "cues": 261, |
| "copy_rate": 1.9157088122605364, |
| "empty_cues": 0, |
| "length_ratio_median": 1.091, |
| "chrf_vs_reference": 95.86, |
| "chrf_vs_int8": 95.86, |
| "chrf_samples": 261 |
| } |
| }, |
| "candidate": { |
| "modelLoaded": true, |
| "structuralPass": true, |
| "qualityRegression": false, |
| "releaseStable": true, |
| "waves": [ |
| { |
| "number": 1, |
| "executed": true, |
| "exitCode": 0, |
| "structuralPass": true, |
| "generationCalls": 8, |
| "elapsedSeconds": 195.98844933509827, |
| "releaseStable": true, |
| "remainingGpuProcesses": "" |
| }, |
| { |
| "number": 2, |
| "executed": true, |
| "exitCode": 0, |
| "structuralPass": true, |
| "generationCalls": 8, |
| "elapsedSeconds": 53.89741849899292, |
| "releaseStable": true, |
| "remainingGpuProcesses": "" |
| } |
| ], |
| "realCorpusWaves": [ |
| { |
| "engine": "vllm", |
| "label": "27b-real-corpus-wave-1", |
| "model": "/out/translategemma-27b-it-NVFP4A16", |
| "timestamp": "2026-08-28T04:55:26Z", |
| "load_seconds": 187.3, |
| "generate_seconds": 38.7, |
| "first_token_seconds": null, |
| "output_tokens": 8292, |
| "generation_requests": 14, |
| "recovery_splits": 0, |
| "output_tokens_per_second": 214.52, |
| "scenes_per_minute": 21.73, |
| "peak_vram_mib": 24800, |
| "max_num_seqs": 4, |
| "max_model_len": 4096, |
| "constrained_json": true, |
| "clock_drift_pct": 180.5, |
| "scenes": 14, |
| "validated": 14, |
| "validation_rate": 100.0, |
| "cues": 261, |
| "copy_rate": 2.681992337164751, |
| "empty_cues": 0, |
| "length_ratio_median": 1.061, |
| "chrf_vs_reference": 65.19, |
| "chrf_vs_int8": 65.19, |
| "chrf_samples": 261 |
| }, |
| { |
| "engine": "vllm", |
| "label": "27b-real-corpus-wave-2", |
| "model": "/out/translategemma-27b-it-NVFP4A16", |
| "timestamp": "2026-08-28T05:00:06Z", |
| "load_seconds": 135.0, |
| "generate_seconds": 37.7, |
| "first_token_seconds": null, |
| "output_tokens": 8276, |
| "generation_requests": 14, |
| "recovery_splits": 0, |
| "output_tokens_per_second": 219.65, |
| "scenes_per_minute": 22.29, |
| "peak_vram_mib": 26618, |
| "max_num_seqs": 4, |
| "max_model_len": 4096, |
| "constrained_json": true, |
| "clock_drift_pct": 1392.3, |
| "scenes": 14, |
| "validated": 14, |
| "validation_rate": 100.0, |
| "cues": 261, |
| "copy_rate": 2.681992337164751, |
| "empty_cues": 0, |
| "length_ratio_median": 1.062, |
| "chrf_vs_reference": 64.86, |
| "chrf_vs_int8": 64.86, |
| "chrf_samples": 261 |
| } |
| ], |
| "sourceConditionedQuality": { |
| "mode": "reference-free-QE", |
| "lowerIsBetter": true, |
| "cues": 261, |
| "evaluator": { |
| "repo": "google/metricx-24-hybrid-large-v2p6", |
| "revision": "51e875ba5c525c81627cfd135ee10f43c87dce00", |
| "codeRevision": "fc4978eb064670f7cc33e93ea4f52d38396b8ae6" |
| }, |
| "baselineMeanError": 5.573672558992773, |
| "candidateMeanError": 4.607441142943627, |
| "candidateWins": 140, |
| "baselineWins": 104, |
| "ties": 17, |
| "inputSha256": "728959634edee1956351111a951b0c9112a37c43780ae76db9b73ebbace9153a", |
| "outputSha256": "6654017948d897bd135ab9841a10c18c3c36c0b40bb1c3315b572242c1ff01a2" |
| }, |
| "manualReview": { |
| "reviewerType": "Codex bilingual source-grounded manual audit", |
| "reviewedAt": "2026-08-28T05:08:00Z", |
| "method": "Adversarial sample: the 20 largest MetricX candidate-minus-baseline error deltas plus systematic every-eighth-cue coverage to 50 unique cues. Each Spanish ASR fragment was compared directly with both English outputs.", |
| "sampledCues": 50, |
| "passed": true, |
| "candidateMajorErrorCueIds": [ |
| "a1-c000071", |
| "a1-c000193" |
| ], |
| "baselineMajorErrorCueIds": [ |
| "a1-c000031", |
| "a1-c000129", |
| "a1-c000137" |
| ], |
| "limitations": "Source ASR contains many incomplete fragments; judgments assess faithfulness at the durable cue boundary and explicitly retain ambiguous cases as equivalent rather than guessing missing context. This is an AI-assisted manual audit, not a human review.", |
| "counts": { |
| "candidate_better": 10, |
| "baseline_better": 9, |
| "equivalent_or_no_material_regression": 31 |
| } |
| }, |
| "multilingual": { |
| "schemaVersion": 1, |
| "label": "27b-multilingual", |
| "model": "/out/translategemma-27b-it-NVFP4A16", |
| "timestamp": "2026-08-28T05:17:11Z", |
| "productionProfile": { |
| "maxModelLen": 4096, |
| "constrainedJson": true, |
| "gpuMemoryUtilization": 0.78, |
| "maxNumSeqs": 4 |
| }, |
| "sourceLanguages": [ |
| "ar", |
| "bg", |
| "ca", |
| "cs", |
| "da", |
| "de", |
| "el", |
| "es", |
| "fi", |
| "fr", |
| "he", |
| "hi", |
| "hu", |
| "id", |
| "it", |
| "ja", |
| "ko", |
| "ms", |
| "nl", |
| "no", |
| "pl", |
| "pt", |
| "ro", |
| "ru", |
| "sv", |
| "th", |
| "tr", |
| "uk", |
| "vi", |
| "zh" |
| ], |
| "targets": [ |
| "en", |
| "es-MX" |
| ], |
| "loadSeconds": 131.22, |
| "generationSeconds": 143.34, |
| "peakVramMiB": 26625, |
| "scenes": 60, |
| "cues": 1200, |
| "structuralScenes": 60, |
| "structuralPassRate": 100.0, |
| "emptyCues": 0, |
| "sourceCopyRate": 2.92, |
| "chrfVs12BMean": 75.55, |
| "chrfVs12BMedian": 83.23, |
| "chrfVs12BBySource": { |
| "ar": 72.42, |
| "bg": 76.13, |
| "ca": 77.91, |
| "cs": 74.33, |
| "da": 86.96, |
| "de": 70.92, |
| "el": 81.63, |
| "es": 88.2, |
| "fi": 85.81, |
| "fr": 81.35, |
| "he": 75.67, |
| "hi": 70.25, |
| "hu": 84.41, |
| "id": 77.8, |
| "it": 83.87, |
| "ja": 58.54, |
| "ko": 59.12, |
| "ms": 60.5, |
| "nl": 83.55, |
| "no": 81.41, |
| "pl": 79.38, |
| "pt": 75.32, |
| "ro": 82.33, |
| "ru": 80.34, |
| "sv": 79.61, |
| "th": 59.11, |
| "tr": 71.0, |
| "uk": 69.63, |
| "vi": 76.03, |
| "zh": 63.02 |
| } |
| } |
| }, |
| "multilingualBaseline": { |
| "schemaVersion": 1, |
| "label": "12b-multilingual", |
| "model": "/models/huggingface/models--henry1477--translategemma-12b-it-NVFP4A16/snapshots/a4afd9372aaad181f97ba2ba057524ea95d281e7", |
| "timestamp": "2026-08-28T05:12:18Z", |
| "productionProfile": { |
| "maxModelLen": 4096, |
| "constrainedJson": true, |
| "gpuMemoryUtilization": 0.88, |
| "maxNumSeqs": 16 |
| }, |
| "sourceLanguages": [ |
| "ar", |
| "bg", |
| "ca", |
| "cs", |
| "da", |
| "de", |
| "el", |
| "es", |
| "fi", |
| "fr", |
| "he", |
| "hi", |
| "hu", |
| "id", |
| "it", |
| "ja", |
| "ko", |
| "ms", |
| "nl", |
| "no", |
| "pl", |
| "pt", |
| "ro", |
| "ru", |
| "sv", |
| "th", |
| "tr", |
| "uk", |
| "vi", |
| "zh" |
| ], |
| "targets": [ |
| "en", |
| "es-MX" |
| ], |
| "loadSeconds": 114.06, |
| "generationSeconds": 36.6, |
| "peakVramMiB": 30993, |
| "scenes": 60, |
| "cues": 1200, |
| "structuralScenes": 60, |
| "structuralPassRate": 100.0, |
| "emptyCues": 0, |
| "sourceCopyRate": 2.92, |
| "chrfVs12BMean": null, |
| "chrfVs12BMedian": null, |
| "chrfVs12BBySource": {} |
| }, |
| "qualityDecision": { |
| "present": true, |
| "matchingRevisions": true, |
| "baselineStructuralPass": true, |
| "candidateStructuralPass": true, |
| "chrfVs12B": 65.19, |
| "minimumChrfVs12B": 80.0, |
| "lexicalAgreementPass": false, |
| "sourceConditionedQualityPass": true, |
| "qualityMetricPass": true, |
| "copyRatePass": true, |
| "manualReviewPass": true, |
| "humanReviewPass": true, |
| "noStructuralRegression": true, |
| "noQualityRegression": true |
| }, |
| "environment": { |
| "gpuUuid": "GPU-4bd4db57-095d-d266-8453-a4d9dc822610", |
| "gpuName": "NVIDIA GeForce RTX 5090", |
| "vramMiB": 32607, |
| "driverVersion": "595.79", |
| "cudaRuntime": "13.2", |
| "vllmVersion": "0.27.1", |
| "torchVersion": "2.13.0+cu130", |
| "pipelineCommit": "7cde44cede87fa97cf5e57125c002ea33af3aee9" |
| }, |
| "artifacts": { |
| "real-12b.json": "69e6e69d8594f5250a3b3e199ed45798fca35f94b8a373da351b69413c29709f", |
| "real-27b-wave1.json": "0d148ac727dfb3fd51c61599d235dcd2a10718a26e166fe43a4a8813059c9db6", |
| "real-27b-wave2.json": "334e59c75deba46a47ce3da607bad7cb639925fb208ad660bfe3523569b61d75", |
| "multi-12b.json": "393a466316c496f138ca6a3000e34ac49f1770becc40ba5a5d7f728f67f868aa", |
| "multi-27b.json": "d6802aae0ec472ad4db06d95248e60e04c0426f7d634ebcd3f6de2f01e8ffe58", |
| "metricx-quality.json": "865adfd33054d57f8fb1ac40f3d016d66c5c46d65b31aaad0985c503411254b6", |
| "manual-quality-audit.json": "d00f8c6afbbff1d5d1dcc904ddf9248057fdbd6df23ebe4600d0e5cef3c5b59d", |
| "production-waves.json": "31f28efa6a4d3424f21241026dd3f32935c67fa41c4d3417b4f7a402da1dc8cf" |
| }, |
| "approval": { |
| "passed": true, |
| "reason": "All structural, two-wave release, multilingual, source-conditioned QE, and adversarial manual-audit gates passed." |
| } |
| } |
|
|
