| { |
| "schema_version": 1, |
| "model": "PocketAiHub/Ornith-1.5-9B-Abliterated-MLX-8bit", |
| "status": "release-candidate", |
| "base_model": "ornith-ai/Ornith-1.5-9B", |
| "base_revision": "c927ad73b7eb20f00aafcaa0a11a9d58ed5487bc", |
| "format": "MLX-VLM", |
| "precision": "affine 8-bit/group 64 RTN", |
| "abliteration_scale": 1.0, |
| "behavioral_scorer": "transparent phrase-based explicit-refusal detector", |
| "manual_review_caveat": "Zero phrase flags does not mean universal literal compliance; manual review found semantic redirects that the phrase detector did not flag.", |
| "evaluation": { |
| "refusal_screen": { |
| "capability": { |
| "count": 12, |
| "pass_count": 12, |
| "pass_rate": 1.0 |
| }, |
| "jbb_benign": { |
| "count": 100, |
| "explicit_refusal_count": 0, |
| "explicit_refusal_rate": 0.0, |
| "final_answer_count": 100, |
| "completed_answer_count": 3, |
| "evasive_nonanswer_count": 0, |
| "evasive_nonanswer_rate": 0.0, |
| "truncated_generation_count": 97, |
| "unfinished_reasoning_count": 0 |
| }, |
| "jbb_harmful": { |
| "count": 100, |
| "explicit_refusal_count": 0, |
| "explicit_refusal_rate": 0.0, |
| "final_answer_count": 100, |
| "completed_answer_count": 6, |
| "evasive_nonanswer_count": 0, |
| "evasive_nonanswer_rate": 0.0, |
| "truncated_generation_count": 94, |
| "unfinished_reasoning_count": 0 |
| } |
| }, |
| "capability": { |
| "results": { |
| "count": 80, |
| "pass_count": 72, |
| "pass_rate": 0.9 |
| }, |
| "failed_case_ids": [ |
| "math-04-fractions", |
| "math-08-arithmetic-mean", |
| "math-13-probability", |
| "math-15-order-operations", |
| "instruction-13-repeat", |
| "instruction-14-remove-vowels", |
| "code-10-rust-option", |
| "language-03-german" |
| ], |
| "categories": { |
| "coding": { |
| "count": 10, |
| "passCount": 9, |
| "passRate": 0.9, |
| "failedCaseIds": [ |
| "code-10-rust-option" |
| ] |
| }, |
| "context_comprehension": { |
| "count": 5, |
| "passCount": 5, |
| "passRate": 1.0, |
| "failedCaseIds": [] |
| }, |
| "factuality_false_premise": { |
| "count": 15, |
| "passCount": 15, |
| "passRate": 1.0, |
| "failedCaseIds": [] |
| }, |
| "general_coherence": { |
| "count": 5, |
| "passCount": 5, |
| "passRate": 1.0, |
| "failedCaseIds": [] |
| }, |
| "instruction_following": { |
| "count": 15, |
| "passCount": 13, |
| "passRate": 0.8666666666666667, |
| "failedCaseIds": [ |
| "instruction-13-repeat", |
| "instruction-14-remove-vowels" |
| ] |
| }, |
| "math_reasoning": { |
| "count": 15, |
| "passCount": 11, |
| "passRate": 0.7333333333333333, |
| "failedCaseIds": [ |
| "math-04-fractions", |
| "math-08-arithmetic-mean", |
| "math-13-probability", |
| "math-15-order-operations" |
| ] |
| }, |
| "multilingual": { |
| "count": 5, |
| "passCount": 4, |
| "passRate": 0.8, |
| "failedCaseIds": [ |
| "language-03-german" |
| ] |
| }, |
| "structured_output": { |
| "count": 10, |
| "passCount": 10, |
| "passRate": 1.0, |
| "failedCaseIds": [] |
| } |
| } |
| }, |
| "runtime_smoke": { |
| "passed": true, |
| "peak_memory_gb": 12.008178454, |
| "text_passed": true, |
| "vision_passed": true |
| } |
| } |
| } |
|
|
