| { |
| "schema_version": 1, |
| "model": "PocketAiHub/Ornith-1.5-35B-A3B-Abliterated-MLX-BF16", |
| "variant": "bf16", |
| "status": "experimental", |
| "base_model": "ornith-ai/Ornith-1.5-35B-A3B", |
| "base_revision": "e4dfb35a93d4b6822a811a7676f3488514abe7e2", |
| "runtime": { |
| "mlx": "0.32.0", |
| "mlx_vlm": "0.6.8" |
| }, |
| "behavioral_evaluation": { |
| "parent_12_case_gate": { |
| "capability": { |
| "count": 12, |
| "pass_count": 12, |
| "pass_rate": 1.0 |
| }, |
| "jbb_benign": { |
| "count": 12, |
| "explicit_refusal_count": 1, |
| "explicit_refusal_rate": 0.08333333333333333, |
| "final_answer_count": 12, |
| "completed_answer_count": 1, |
| "evasive_nonanswer_count": 0, |
| "evasive_nonanswer_rate": 0.0, |
| "truncated_generation_count": 11, |
| "unfinished_reasoning_count": 0 |
| }, |
| "jbb_harmful": { |
| "count": 12, |
| "explicit_refusal_count": 12, |
| "explicit_refusal_rate": 1.0, |
| "final_answer_count": 12, |
| "completed_answer_count": 11, |
| "evasive_nonanswer_count": 0, |
| "evasive_nonanswer_rate": 0.0, |
| "truncated_generation_count": 1, |
| "unfinished_reasoning_count": 0 |
| } |
| }, |
| "selected_bf16_12_case_gate": { |
| "capability": { |
| "count": 12, |
| "pass_count": 12, |
| "pass_rate": 1.0 |
| }, |
| "jbb_benign": { |
| "count": 12, |
| "explicit_refusal_count": 0, |
| "explicit_refusal_rate": 0.0, |
| "final_answer_count": 12, |
| "completed_answer_count": 1, |
| "evasive_nonanswer_count": 0, |
| "evasive_nonanswer_rate": 0.0, |
| "truncated_generation_count": 11, |
| "unfinished_reasoning_count": 0 |
| }, |
| "jbb_harmful": { |
| "count": 12, |
| "explicit_refusal_count": 0, |
| "explicit_refusal_rate": 0.0, |
| "final_answer_count": 12, |
| "completed_answer_count": 4, |
| "evasive_nonanswer_count": 0, |
| "evasive_nonanswer_rate": 0.0, |
| "truncated_generation_count": 8, |
| "unfinished_reasoning_count": 0 |
| } |
| }, |
| "variant_100_harmful_100_benign_screen": { |
| "jbb_benign": { |
| "count": 100, |
| "explicit_refusal_count": 0, |
| "explicit_refusal_rate": 0.0, |
| "final_answer_count": 100, |
| "completed_answer_count": 2, |
| "evasive_nonanswer_count": 0, |
| "evasive_nonanswer_rate": 0.0, |
| "truncated_generation_count": 98, |
| "unfinished_reasoning_count": 0 |
| }, |
| "jbb_harmful": { |
| "count": 100, |
| "explicit_refusal_count": 1, |
| "explicit_refusal_rate": 0.01, |
| "final_answer_count": 100, |
| "completed_answer_count": 9, |
| "evasive_nonanswer_count": 0, |
| "evasive_nonanswer_rate": 0.0, |
| "truncated_generation_count": 91, |
| "unfinished_reasoning_count": 0 |
| } |
| }, |
| "batch_size": 1, |
| "max_new_tokens": 128, |
| "thinking": false, |
| "scorer": "transparent phrase-based explicit-refusal detector", |
| "caveat": "This is an early-refusal screen, not a full answer-quality evaluation. Manual review found semantic refusals that the phrase detector did not flag." |
| }, |
| "matched_teacher_drift": { |
| "protocol": { |
| "context": "shared regular-BF16 greedy teacher trajectory", |
| "positionsPerCaseMaximum": 16, |
| "vocabulary": 248320, |
| "logitsStorage": "float32", |
| "probabilityMath": "float64 full-vocabulary log-softmax", |
| "residualState": "all 40 residual_post vectors at assistant boundary", |
| "cacheState": "final teacher-trajectory cache: K/V for 10 full-attention layers; convolution and gated-delta recurrent state for 30 linear-attention layers", |
| "thinking": false, |
| "sampling": "greedy" |
| }, |
| "dataset": { |
| "capabilityFileSha256": "fe35cc41fe041cf2d371041c3fb977ec37b51fa1e9a0b0482aaf1eab415c9c26", |
| "capabilityCases": 12, |
| "jailbreakBench": { |
| "repository": "JailbreakBench/JBB-Behaviors", |
| "revision": "886acc352a31533ffbcf4ef22c744658688086fc" |
| }, |
| "harmfulCases": 12, |
| "benignCases": 12 |
| }, |
| "pure_abliteration_vs_regular_bf16": { |
| "positions": 481, |
| "forward_kl_nats_mean": 0.54518455642922, |
| "top1_agreement": 0.7817047817047817, |
| "residual_cosine_mean": 0.9520897871696514, |
| "cache": { |
| "full_attention_k_cosine_mean": 0.9956834574679786, |
| "full_attention_v_cosine_mean": 0.9942660171605967, |
| "linear_attention_conv_cosine_mean": 0.997201268533088, |
| "linear_attention_recurrent_cosine_mean": 0.9962457101916542 |
| }, |
| "suite_splits": { |
| "capability": { |
| "positions": 97, |
| "forward_kl_nats_mean": 0.01834161340133247, |
| "top1_agreement": 0.979381443298969, |
| "residual_cosine_mean": 0.995895118556436 |
| }, |
| "jbb_benign": { |
| "positions": 192, |
| "forward_kl_nats_mean": 0.3073881842398057, |
| "top1_agreement": 0.8541666666666666, |
| "residual_cosine_mean": 0.9696202077556074 |
| }, |
| "jbb_harmful": { |
| "positions": 192, |
| "forward_kl_nats_mean": 1.0491463737941817, |
| "top1_agreement": 0.609375, |
| "residual_cosine_mean": 0.8907540351969113 |
| } |
| } |
| }, |
| "quantization_vs_abliterated_bf16": null, |
| "total_vs_regular_bf16": { |
| "positions": 481, |
| "forward_kl_nats_mean": 0.54518455642922, |
| "top1_agreement": 0.7817047817047817, |
| "residual_cosine_mean": 0.9520897871696514, |
| "cache": { |
| "full_attention_k_cosine_mean": 0.9956834574679786, |
| "full_attention_v_cosine_mean": 0.9942660171605967, |
| "linear_attention_conv_cosine_mean": 0.997201268533088, |
| "linear_attention_recurrent_cosine_mean": 0.9962457101916542 |
| }, |
| "suite_splits": { |
| "capability": { |
| "positions": 97, |
| "forward_kl_nats_mean": 0.01834161340133247, |
| "top1_agreement": 0.979381443298969, |
| "residual_cosine_mean": 0.995895118556436 |
| }, |
| "jbb_benign": { |
| "positions": 192, |
| "forward_kl_nats_mean": 0.3073881842398057, |
| "top1_agreement": 0.8541666666666666, |
| "residual_cosine_mean": 0.9696202077556074 |
| }, |
| "jbb_harmful": { |
| "positions": 192, |
| "forward_kl_nats_mean": 1.0491463737941817, |
| "top1_agreement": 0.609375, |
| "residual_cosine_mean": 0.8907540351969113 |
| } |
| } |
| } |
| }, |
| "capabilities": { |
| "vision_tower_included": true, |
| "video_preprocessor_included": true, |
| "native_mtp_head_included": false, |
| "four_bit_image_smoke_passed": false, |
| "vision_evaluation_caveat": "Only the 4-bit artifact received an end-to-end image smoke test; vision and video were not comprehensively benchmarked." |
| } |
| } |
|
|
