File size: 3,591 Bytes
3b98225 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 | {
"schema_version": 1,
"model": "PocketAiHub/Ornith-1.5-9B-Abliterated-MLX-8bit",
"status": "release-candidate",
"base_model": "ornith-ai/Ornith-1.5-9B",
"base_revision": "c927ad73b7eb20f00aafcaa0a11a9d58ed5487bc",
"format": "MLX-VLM",
"precision": "affine 8-bit/group 64 RTN",
"abliteration_scale": 1.0,
"behavioral_scorer": "transparent phrase-based explicit-refusal detector",
"manual_review_caveat": "Zero phrase flags does not mean universal literal compliance; manual review found semantic redirects that the phrase detector did not flag.",
"evaluation": {
"refusal_screen": {
"capability": {
"count": 12,
"pass_count": 12,
"pass_rate": 1.0
},
"jbb_benign": {
"count": 100,
"explicit_refusal_count": 0,
"explicit_refusal_rate": 0.0,
"final_answer_count": 100,
"completed_answer_count": 3,
"evasive_nonanswer_count": 0,
"evasive_nonanswer_rate": 0.0,
"truncated_generation_count": 97,
"unfinished_reasoning_count": 0
},
"jbb_harmful": {
"count": 100,
"explicit_refusal_count": 0,
"explicit_refusal_rate": 0.0,
"final_answer_count": 100,
"completed_answer_count": 6,
"evasive_nonanswer_count": 0,
"evasive_nonanswer_rate": 0.0,
"truncated_generation_count": 94,
"unfinished_reasoning_count": 0
}
},
"capability": {
"results": {
"count": 80,
"pass_count": 72,
"pass_rate": 0.9
},
"failed_case_ids": [
"math-04-fractions",
"math-08-arithmetic-mean",
"math-13-probability",
"math-15-order-operations",
"instruction-13-repeat",
"instruction-14-remove-vowels",
"code-10-rust-option",
"language-03-german"
],
"categories": {
"coding": {
"count": 10,
"passCount": 9,
"passRate": 0.9,
"failedCaseIds": [
"code-10-rust-option"
]
},
"context_comprehension": {
"count": 5,
"passCount": 5,
"passRate": 1.0,
"failedCaseIds": []
},
"factuality_false_premise": {
"count": 15,
"passCount": 15,
"passRate": 1.0,
"failedCaseIds": []
},
"general_coherence": {
"count": 5,
"passCount": 5,
"passRate": 1.0,
"failedCaseIds": []
},
"instruction_following": {
"count": 15,
"passCount": 13,
"passRate": 0.8666666666666667,
"failedCaseIds": [
"instruction-13-repeat",
"instruction-14-remove-vowels"
]
},
"math_reasoning": {
"count": 15,
"passCount": 11,
"passRate": 0.7333333333333333,
"failedCaseIds": [
"math-04-fractions",
"math-08-arithmetic-mean",
"math-13-probability",
"math-15-order-operations"
]
},
"multilingual": {
"count": 5,
"passCount": 4,
"passRate": 0.8,
"failedCaseIds": [
"language-03-german"
]
},
"structured_output": {
"count": 10,
"passCount": 10,
"passRate": 1.0,
"failedCaseIds": []
}
}
},
"runtime_smoke": {
"passed": true,
"peak_memory_gb": 12.008178454,
"text_passed": true,
"vision_passed": true
}
}
}
|