| { |
| "checkpoint": "/Volumes/ExternalDisk/UserData/code/loadchange/qev/models/qev-snake-0.8b-mlx", |
| "backend": "mlx", |
| "base_model": "Qwen/Qwen3.5-0.8B", |
| "base_revision": "2fc06364715b967f1860aea9cf38778875588b17", |
| "config_sha256": "72608203d8498b14ecc8975980fcf0aef52c11e695238f9af597a7ec75296697", |
| "checks": { |
| "health": { |
| "passed": true, |
| "seconds": 0.0012778750387951732, |
| "result": { |
| "status": "ok", |
| "backend": "mlx" |
| } |
| }, |
| "models_http": { |
| "passed": true, |
| "seconds": 0.0007342080352827907, |
| "result": { |
| "models": [ |
| { |
| "name": "qev-0.8b", |
| "description": "Qwen3.5 typed decisions; multimodal decision accuracy is not yet validated", |
| "release_date": "2026-09-20" |
| }, |
| { |
| "name": "qev-latest", |
| "description": "Alias of qev-0.8b", |
| "release_date": "2026-09-20" |
| }, |
| { |
| "name": "qev:0.8b", |
| "description": "Alias of loaded checkpoint qev-0.8b", |
| "release_date": "2026-09-20" |
| }, |
| { |
| "name": "qev:0.8b-mlx", |
| "description": "Alias of loaded checkpoint qev-0.8b; aliases do not switch the runtime", |
| "release_date": "2026-09-20" |
| }, |
| { |
| "name": "qev-native", |
| "description": "Native Qwen text/image/video generation with the decision adapter disabled", |
| "release_date": "2026-09-20" |
| } |
| ] |
| } |
| }, |
| "text_decision": { |
| "passed": true, |
| "seconds": 0.16025920800166205, |
| "result": { |
| "model": "jev-latest", |
| "answers": { |
| "department": { |
| "type": "choice", |
| "choice": "billing", |
| "confidence": 0.9987527358686498, |
| "probabilities": { |
| "billing": 0.9991684905791, |
| "technical": 0.0003443647363560615, |
| "sales": 0.0004871446845439485 |
| } |
| }, |
| "refund": { |
| "type": "noul", |
| "noul": 0.9857972855489886 |
| }, |
| "urgency": { |
| "type": "score", |
| "score": 1.4681707822149208, |
| "confidence": 0.7340853911074603, |
| "legend": { |
| "0": "No time pressure", |
| "1": "Soon", |
| "2": "Today or a blocking issue" |
| }, |
| "probabilities": { |
| "0": 0.17230730721222765, |
| "1": 0.18721460336062398, |
| "2": 0.6404780894271485 |
| } |
| } |
| }, |
| "usage": { |
| "input_tokens": 172, |
| "output_tokens": 0 |
| }, |
| "latency_ms": 158.84, |
| "qev": { |
| "backend": "mlx", |
| "temperature": 2.82842712474619, |
| "training_modalities": [ |
| "text" |
| ], |
| "multimodal_decision_accuracy_validated": false, |
| "input_modalities": [ |
| "text" |
| ], |
| "truncated_questions": [] |
| } |
| } |
| }, |
| "official_sdk_over_http": { |
| "passed": true, |
| "seconds": 0.16014291701139882, |
| "result": { |
| "response": { |
| "model": "jev-latest", |
| "usage": { |
| "input_tokens": 172, |
| "output_tokens": 0 |
| }, |
| "answers": { |
| "department": { |
| "type": "choice", |
| "choice": "billing", |
| "confidence": 0.9987527358686498, |
| "probabilities": { |
| "billing": 0.9991684905791, |
| "technical": 0.0003443647363560615, |
| "sales": 0.0004871446845439485 |
| } |
| }, |
| "refund": { |
| "type": "noul", |
| "noul": 0.9857972855489886 |
| }, |
| "urgency": { |
| "type": "score", |
| "score": 1.4681707822149208, |
| "confidence": 0.7340853911074603, |
| "legend": { |
| "0": "No time pressure", |
| "1": "Soon", |
| "2": "Today or a blocking issue" |
| }, |
| "probabilities": { |
| "0": 0.17230730721222765, |
| "1": 0.18721460336062398, |
| "2": 0.6404780894271485 |
| } |
| } |
| } |
| }, |
| "comparison": { |
| "max_probability_error": 0.0, |
| "atol": 1e-05 |
| }, |
| "typed_accessors": true |
| } |
| }, |
| "typed_model_aliases": { |
| "passed": true, |
| "seconds": 0.6996553330100141, |
| "result": { |
| "qev-latest": { |
| "max_probability_error": 0.0, |
| "atol": 1e-05 |
| }, |
| "qev:0.8b": { |
| "max_probability_error": 0.0, |
| "atol": 1e-05 |
| }, |
| "qev:0.8b-mlx": { |
| "max_probability_error": 0.0, |
| "atol": 1e-05 |
| }, |
| "qev-0.8b": { |
| "max_probability_error": 0.0, |
| "atol": 1e-05 |
| }, |
| "qev-0.8b-mlx": { |
| "max_probability_error": 0.0, |
| "atol": 1e-05 |
| } |
| } |
| }, |
| "independent_question_rows": { |
| "passed": true, |
| "seconds": 0.29000495799118653, |
| "result": { |
| "department": { |
| "max_probability_error": 0.0, |
| "atol": 0.005 |
| }, |
| "refund": { |
| "max_probability_error": 0.0, |
| "atol": 0.005 |
| }, |
| "urgency": { |
| "max_probability_error": 0.0, |
| "atol": 0.005 |
| }, |
| "reversed": { |
| "max_probability_error": 0.0, |
| "atol": 0.005 |
| } |
| } |
| }, |
| "native_text_before_media": { |
| "passed": true, |
| "seconds": 0.32884416601154953, |
| "result": { |
| "id": "chatcmpl-b9aff1dcc9bf46dca581157908d126ca", |
| "object": "chat.completion", |
| "created": 1789951846, |
| "model": "qev-native", |
| "choices": [ |
| { |
| "index": 0, |
| "message": { |
| "role": "assistant", |
| "content": "ready\n" |
| }, |
| "finish_reason": "stop" |
| } |
| ], |
| "usage": { |
| "prompt_tokens": 19, |
| "completion_tokens": 4, |
| "total_tokens": 23 |
| }, |
| "qev": { |
| "backend": "mlx", |
| "decision_adapter_enabled": false, |
| "media_frames": 0, |
| "latency_ms": 326.94 |
| } |
| } |
| }, |
| "native_sampling": { |
| "passed": true, |
| "seconds": 0.15504195797257125, |
| "result": { |
| "id": "chatcmpl-b94bbadcc5fc4fcaa557c6d70604a87f", |
| "object": "chat.completion", |
| "created": 1789951846, |
| "model": "qev-native", |
| "choices": [ |
| { |
| "index": 0, |
| "message": { |
| "role": "assistant", |
| "content": "ready\n" |
| }, |
| "finish_reason": "stop" |
| } |
| ], |
| "usage": { |
| "prompt_tokens": 19, |
| "completion_tokens": 4, |
| "total_tokens": 23 |
| }, |
| "qev": { |
| "backend": "mlx", |
| "decision_adapter_enabled": false, |
| "media_frames": 0, |
| "latency_ms": 153.04 |
| } |
| } |
| }, |
| "image_decision": { |
| "passed": true, |
| "seconds": 0.13663308299146593, |
| "result": { |
| "model": "jev-latest", |
| "answers": { |
| "color": { |
| "type": "choice", |
| "choice": "red", |
| "confidence": 0.9999999999344191, |
| "probabilities": { |
| "red": 0.9999999999562794, |
| "blue": 3.5183963380108156e-11, |
| "green": 8.536683512487626e-12 |
| } |
| } |
| }, |
| "usage": { |
| "input_tokens": 96, |
| "output_tokens": 0 |
| }, |
| "latency_ms": 134.95, |
| "qev": { |
| "backend": "mlx", |
| "temperature": 1.0, |
| "training_modalities": [ |
| "text" |
| ], |
| "multimodal_decision_accuracy_validated": false, |
| "input_modalities": [ |
| "text", |
| "image" |
| ], |
| "truncated_questions": [] |
| } |
| } |
| }, |
| "video_decision": { |
| "passed": true, |
| "seconds": 0.15981079201446846, |
| "result": { |
| "model": "jev-latest", |
| "answers": { |
| "motion": { |
| "type": "choice", |
| "choice": "left to right", |
| "confidence": 0.9983157042043018, |
| "probabilities": { |
| "left to right": 0.9988771361362013, |
| "right to left": 0.00024130962942561419, |
| "stationary": 0.0008815542343730066 |
| } |
| } |
| }, |
| "usage": { |
| "input_tokens": 145, |
| "output_tokens": 0 |
| }, |
| "latency_ms": 157.98, |
| "qev": { |
| "backend": "mlx", |
| "temperature": 1.0, |
| "training_modalities": [ |
| "text" |
| ], |
| "multimodal_decision_accuracy_validated": false, |
| "input_modalities": [ |
| "text", |
| "video" |
| ], |
| "truncated_questions": [] |
| } |
| } |
| }, |
| "mixed_image_video_decision": { |
| "passed": true, |
| "seconds": 0.21332191699184477, |
| "result": { |
| "model": "jev-latest", |
| "answers": { |
| "color": { |
| "type": "choice", |
| "choice": "red", |
| "confidence": 0.9999999974323013, |
| "probabilities": { |
| "red": 0.9999999982882009, |
| "blue": 1.6035388545083474e-09, |
| "green": 1.0826019229201883e-10 |
| } |
| } |
| }, |
| "usage": { |
| "input_tokens": 211, |
| "output_tokens": 0 |
| }, |
| "latency_ms": 211.74, |
| "qev": { |
| "backend": "mlx", |
| "temperature": 1.0, |
| "training_modalities": [ |
| "text" |
| ], |
| "multimodal_decision_accuracy_validated": false, |
| "input_modalities": [ |
| "text", |
| "image", |
| "video" |
| ], |
| "truncated_questions": [] |
| } |
| } |
| }, |
| "native_image": { |
| "passed": true, |
| "seconds": 0.21933616697788239, |
| "result": { |
| "id": "chatcmpl-903c9409484740a089d1eebd81255f1a", |
| "object": "chat.completion", |
| "created": 1789951847, |
| "model": "qev-native", |
| "choices": [ |
| { |
| "index": 0, |
| "message": { |
| "role": "assistant", |
| "content": " red\n" |
| }, |
| "finish_reason": "stop" |
| } |
| ], |
| "usage": { |
| "prompt_tokens": 92, |
| "completion_tokens": 4, |
| "total_tokens": 96 |
| }, |
| "qev": { |
| "backend": "mlx", |
| "decision_adapter_enabled": false, |
| "media_frames": 1, |
| "latency_ms": 216.99 |
| } |
| } |
| }, |
| "native_video": { |
| "passed": true, |
| "seconds": 0.594233791984152, |
| "result": { |
| "id": "chatcmpl-1e20b76b0aa04309b4c5a5d329f78e21", |
| "object": "chat.completion", |
| "created": 1789951848, |
| "model": "qev-native", |
| "choices": [ |
| { |
| "index": 0, |
| "message": { |
| "role": "assistant", |
| "content": "The blue circle moves from left to right.\n" |
| }, |
| "finish_reason": "stop" |
| } |
| ], |
| "usage": { |
| "prompt_tokens": 140, |
| "completion_tokens": 12, |
| "total_tokens": 152 |
| }, |
| "qev": { |
| "backend": "mlx", |
| "decision_adapter_enabled": false, |
| "media_frames": 4, |
| "latency_ms": 591.23 |
| } |
| } |
| }, |
| "native_text_after_media": { |
| "passed": true, |
| "seconds": 0.16757304198108613, |
| "result": { |
| "identical_continuation_and_usage": true, |
| "response": { |
| "id": "chatcmpl-7c88f513d018444ebb6f67ed7a8e7928", |
| "object": "chat.completion", |
| "created": 1789951848, |
| "model": "qev-native", |
| "choices": [ |
| { |
| "index": 0, |
| "message": { |
| "role": "assistant", |
| "content": "ready\n" |
| }, |
| "finish_reason": "stop" |
| } |
| ], |
| "usage": { |
| "prompt_tokens": 19, |
| "completion_tokens": 4, |
| "total_tokens": 23 |
| }, |
| "qev": { |
| "backend": "mlx", |
| "decision_adapter_enabled": false, |
| "media_frames": 0, |
| "latency_ms": 164.33 |
| } |
| } |
| } |
| }, |
| "decision_adapter_restored": { |
| "passed": true, |
| "seconds": 0.2970623329747468, |
| "result": { |
| "max_probability_error": 0.0, |
| "atol": 1e-05 |
| } |
| }, |
| "concurrent_adapter_modes": { |
| "passed": true, |
| "seconds": 0.28654124995227903, |
| "result": { |
| "decision": { |
| "max_probability_error": 0.0, |
| "atol": 1e-05 |
| }, |
| "native_identical": true |
| } |
| }, |
| "invalid_inputs": { |
| "passed": true, |
| "seconds": 0.11006774997804314, |
| "result": { |
| "unknown_model": { |
| "status": 404, |
| "detail": "Unknown model alias" |
| }, |
| "remote_image": { |
| "status": 422, |
| "detail": "images must be inline base64 PNG/JPEG/WebP data URLs; paths and remote URLs are not supported" |
| }, |
| "invalid_fps": { |
| "status": 422, |
| "detail": "video fps must be finite and between zero and 60" |
| }, |
| "too_many_frames": { |
| "status": 422, |
| "detail": "too many images/video frames in one request" |
| }, |
| "invalid_score": { |
| "status": 422, |
| "detail": [ |
| { |
| "type": "too_short", |
| "loc": [ |
| "body", |
| "questions", |
| "q", |
| "score", |
| "criteria" |
| ], |
| "msg": "List should have at least 2 items after validation, not 1", |
| "input": [ |
| "only" |
| ], |
| "ctx": { |
| "field_type": "List", |
| "min_length": 2, |
| "actual_length": 1 |
| } |
| } |
| ] |
| }, |
| "too_many_questions": { |
| "status": 422, |
| "detail": "At most 64 questions per request" |
| }, |
| "streaming": { |
| "status": 422, |
| "detail": [ |
| { |
| "type": "literal_error", |
| "loc": [ |
| "body", |
| "stream" |
| ], |
| "msg": "Input should be False", |
| "input": true, |
| "ctx": { |
| "expected": "False" |
| } |
| } |
| ] |
| }, |
| "conflicting_budgets": { |
| "status": 422, |
| "detail": [ |
| { |
| "type": "value_error", |
| "loc": [ |
| "body" |
| ], |
| "msg": "Value error, set max_tokens or max_completion_tokens, not both", |
| "input": { |
| "model": "qev-native", |
| "messages": [ |
| { |
| "role": "user", |
| "content": "Reply with the single word ready." |
| } |
| ], |
| "temperature": 0, |
| "max_tokens": 16, |
| "enable_thinking": false, |
| "max_completion_tokens": 16 |
| }, |
| "ctx": { |
| "error": {} |
| } |
| } |
| ] |
| }, |
| "native_path": { |
| "status": 422, |
| "detail": "images must be inline base64 PNG/JPEG/WebP data URLs; paths and remote URLs are not supported" |
| }, |
| "oversized_body": { |
| "status": 413 |
| } |
| } |
| }, |
| "healthy_after_rejections": { |
| "passed": true, |
| "seconds": 0.0007556669879704714, |
| "result": { |
| "status": "ok", |
| "backend": "mlx" |
| } |
| }, |
| "inference_after_rejections": { |
| "passed": true, |
| "seconds": 0.13404716597869992, |
| "result": { |
| "max_probability_error": 0.0, |
| "atol": 1e-05 |
| } |
| } |
| }, |
| "passed": true, |
| "scope": "Real checkpoint via HTTP and the official TypeSafe SDK. Checks schemas, media plumbing, question isolation, adapter restoration, and repeated greedy text. These are functional checks, not held-out accuracy or cross-runtime native parity.", |
| "artifact_sha256": { |
| "pointer.safetensors": "c4e7d9439a101a4a398f98932e75fa5c92396fd9101d156b0f8af66694fff66a", |
| "decision_adapters.safetensors": "4d27561fc569dceed431d6e80db7146b215a9e9943064c598345bcecfcadf6f8" |
| }, |
| "versions": { |
| "typesafe-sdk": "0.7.0", |
| "httpx": "0.28.1", |
| "transformers": "5.17.0", |
| "torch": "2.14.0", |
| "mlx": "0.32.2", |
| "mlx-vlm": "0.7.1" |
| }, |
| "server_log": "/Volumes/ExternalDisk/UserData/code/loadchange/qev/docs/results/snake_training/service_mlx.server.log", |
| "server_command": [ |
| "/Volumes/ExternalDisk/UserData/code/loadchange/qev/.venv/bin/python", |
| "-m", |
| "qev.cli", |
| "serve", |
| "--model", |
| "/Volumes/ExternalDisk/UserData/code/loadchange/qev/models/qev-snake-0.8b-mlx", |
| "--backend", |
| "mlx", |
| "--host", |
| "127.0.0.1", |
| "--port", |
| "55947" |
| ], |
| "base_url": "http://127.0.0.1:55947", |
| "seconds": 8.822781457973178 |
| } |
|
|
