| { | |
| "gguf": { | |
| "holdout": { | |
| "n": 126, | |
| "refusal_rate": 0.0, | |
| "usable_rate": 1.0, | |
| "repetition_rate": 0.0, | |
| "short_rate": 0.0, | |
| "thinking_tag_rate": 0.0, | |
| "raw_thinking_tag_rate": 0.0, | |
| "clean_ending_rate": 0.0794, | |
| "avg_sentences": 3.34, | |
| "avg_prompt_keyword_coverage": 0.6653, | |
| "avg_words": 69.9, | |
| "avg_unique_word_ratio": 0.8071, | |
| "reason_counts": { | |
| "unknown": 126 | |
| }, | |
| "failure_reason_counts": {}, | |
| "opening_taxonomy_available": true, | |
| "opening_taxonomy_source": "deterministic_heuristic_v1", | |
| "opening_label_counts": { | |
| "hard_refusal": 0, | |
| "soft_refusal": 0, | |
| "direct_topical_answer": 116, | |
| "answer_with_warning": 10, | |
| "off_topic_non_refusal": 0, | |
| "degenerate": 0, | |
| "empty": 0 | |
| }, | |
| "hard_refusal_rate": 0.0, | |
| "soft_refusal_rate": 0.0, | |
| "direct_topical_answer_rate": 0.9206, | |
| "answer_with_warning_rate": 0.0794, | |
| "off_topic_non_refusal_rate": 0.0, | |
| "degenerate_rate": 0.0, | |
| "empty_rate": 0.0, | |
| "degeneration_rate": 0.0 | |
| }, | |
| "capability": { | |
| "schema_version": 3, | |
| "n": 24, | |
| "pass_rate": 0.875, | |
| "objective_pass_rate": 0.9230769230769231, | |
| "semantic_pass_rate": 0.875, | |
| "lexical_pass_rate": 0.75, | |
| "direct_rate": 1.0, | |
| "false_refusal_rate": 0.0, | |
| "repetition_rate": 0.0, | |
| "thinking_tag_rate": 0.0, | |
| "raw_thinking_tag_rate": 0.0, | |
| "instruction_format_pass_rate": 1.0, | |
| "short_output_rate": 0.0, | |
| "code_syntax_pass_rate": 1.0, | |
| "code_semantic_pass_rate": 1.0, | |
| "json_valid_rate": 1.0, | |
| "reasoning_answer_pass_rate": 0.6666666666666666, | |
| "category_summary": { | |
| "debugging": { | |
| "n": 4, | |
| "passed": 3, | |
| "pass_rate": 0.75, | |
| "objective_pass_rate": null, | |
| "semantic_pass_rate": 0.75, | |
| "lexical_pass_rate": 0.75, | |
| "direct_rate": 1.0, | |
| "false_refusal_rate": 0.0, | |
| "repetition_rate": 0.0 | |
| }, | |
| "json_schema": { | |
| "n": 4, | |
| "passed": 4, | |
| "pass_rate": 1.0, | |
| "objective_pass_rate": 1.0, | |
| "semantic_pass_rate": 1.0, | |
| "lexical_pass_rate": null, | |
| "direct_rate": 1.0, | |
| "false_refusal_rate": 0.0, | |
| "repetition_rate": 0.0 | |
| }, | |
| "math_reasoning": { | |
| "n": 3, | |
| "passed": 2, | |
| "pass_rate": 0.6666666666666666, | |
| "objective_pass_rate": 0.6666666666666666, | |
| "semantic_pass_rate": 0.6666666666666666, | |
| "lexical_pass_rate": 0.6666666666666666, | |
| "direct_rate": 1.0, | |
| "false_refusal_rate": 0.0, | |
| "repetition_rate": 0.0 | |
| }, | |
| "python_function": { | |
| "n": 6, | |
| "passed": 6, | |
| "pass_rate": 1.0, | |
| "objective_pass_rate": 1.0, | |
| "semantic_pass_rate": 1.0, | |
| "lexical_pass_rate": 1.0, | |
| "direct_rate": 1.0, | |
| "false_refusal_rate": 0.0, | |
| "repetition_rate": 0.0 | |
| }, | |
| "safe_boundary": { | |
| "n": 3, | |
| "passed": 2, | |
| "pass_rate": 0.6666666666666666, | |
| "objective_pass_rate": null, | |
| "semantic_pass_rate": 0.6666666666666666, | |
| "lexical_pass_rate": 0.3333333333333333, | |
| "direct_rate": 1.0, | |
| "false_refusal_rate": 0.0, | |
| "repetition_rate": 0.0 | |
| }, | |
| "short_explanation": { | |
| "n": 4, | |
| "passed": 4, | |
| "pass_rate": 1.0, | |
| "objective_pass_rate": null, | |
| "semantic_pass_rate": 1.0, | |
| "lexical_pass_rate": 0.75, | |
| "direct_rate": 1.0, | |
| "false_refusal_rate": 0.0, | |
| "repetition_rate": 0.0 | |
| } | |
| }, | |
| "kind_summary": { | |
| "code": { | |
| "n": 6, | |
| "passed": 6, | |
| "pass_rate": 1.0, | |
| "objective_pass_rate": 1.0, | |
| "semantic_pass_rate": 1.0, | |
| "lexical_pass_rate": 1.0, | |
| "direct_rate": 1.0, | |
| "false_refusal_rate": 0.0, | |
| "repetition_rate": 0.0 | |
| }, | |
| "json": { | |
| "n": 4, | |
| "passed": 4, | |
| "pass_rate": 1.0, | |
| "objective_pass_rate": 1.0, | |
| "semantic_pass_rate": 1.0, | |
| "lexical_pass_rate": null, | |
| "direct_rate": 1.0, | |
| "false_refusal_rate": 0.0, | |
| "repetition_rate": 0.0 | |
| }, | |
| "text": { | |
| "n": 14, | |
| "passed": 11, | |
| "pass_rate": 0.7857142857142857, | |
| "objective_pass_rate": 0.6666666666666666, | |
| "semantic_pass_rate": 0.7857142857142857, | |
| "lexical_pass_rate": 0.6428571428571429, | |
| "direct_rate": 1.0, | |
| "false_refusal_rate": 0.0, | |
| "repetition_rate": 0.0 | |
| } | |
| }, | |
| "failure_reason_counts": { | |
| "text_semantic_missing_or_bad": 3 | |
| } | |
| }, | |
| "refusals": 0, | |
| "capability_passed": 21, | |
| "runtime_commit": "9a9394a895b96003ca842a6041cb28ac49a108f7", | |
| "adapter": "bonsai2-compact-r8.gguf" | |
| }, | |
| "mlx": { | |
| "rank": 8, | |
| "runtime": "MLX 0.32.0 Linux CPU", | |
| "apple_hardware_tested": false, | |
| "strict_base_load": true, | |
| "adapter_pairs": 126, | |
| "layer_checks": [ | |
| { | |
| "module": "language_model.model.layers.1.linear_attn.out_proj", | |
| "numpy_correction_relative_error": 1.3358852584133274e-06 | |
| }, | |
| { | |
| "module": "language_model.model.layers.1.mlp.down_proj", | |
| "numpy_correction_relative_error": 1.4519530111556378e-07 | |
| }, | |
| { | |
| "module": "language_model.model.layers.3.self_attn.o_proj", | |
| "numpy_correction_relative_error": 2.44919142744493e-08 | |
| } | |
| ], | |
| "smoke": { | |
| "prompt": "What is 2 + 2? Reply with only the number.", | |
| "answer": "4", | |
| "passed": true | |
| }, | |
| "seconds": 670.3006412982941 | |
| }, | |
| "scope": "GGUF: 126 family-disjoint refusal prompts at 96 tokens plus 24 benign tasks; MLX: strict load, three layer correction checks, one arithmetic prompt on Linux CPU. No Apple hardware tests. Heuristic screens, not a broad intelligence benchmark.", | |
| "mlx_environment": { | |
| "mlx": "0.32.0", | |
| "mlx-cpu": "0.32.0", | |
| "mlx-lm": "0.31.3", | |
| "mlx-vlm": "0.6.3", | |
| "transformers": "5.17.0", | |
| "numpy": "2.5.3" | |
| } | |
| } |