Release TinyQuery 139.7M from scratch with frozen weights, reproducible Mac evaluations and runtime source

LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2026 Alibaba Cloud
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -0,0 +1,111 @@
+---
+language:
+- en
+- hi
+license: apache-2.0
+library_name: pytorch
+tags:
+- text-to-sql
+- tool-calling
+- mcp
+- from-scratch
+- tiny-model
+- custom-code
+- safetensors
+datasets:
+- karmx/TinyQuery-Tools-Multilingual
+---
+
+# TinyQuery-140M
+
+**139.7M parameters, trained from random initialization on one RTX PRO 6000 within a four-hour experiment.** TinyQuery turns a question, database schema and runtime tool definitions into a JSON action. It targets English, imperfect English, Hindi and Hinglish, with bounded read-only MySQL and PostgreSQL/Supabase queries and MCP-style tool calls.
+
+The frozen student achieved **92.47% full task success (1,106/1,196)** on the held-out test and **82.50% (132/160)** on additional handwritten-phrasing cases. It is an experimental narrow model, not a general chatbot or a guarantee of correct SQL.
+
+This repository contains the weights, training/inference source, tokenizer, configurations, evaluation reports, architecture documentation and runtime recipes. **This is a custom native PyTorch checkpoint. Transformers `AutoModel` and `pipeline()` cannot load it.**
+
+## Run with streaming output
+
+Requires Python 3.12. Download this repository, then run from its directory:
+
+```sh
+python3.12 -m venv .venv
+source .venv/bin/activate
+python -m pip install -r requirements.txt
+python -m tinyquery.chat "दिल्ली के ग्राहकों के नाम दिखाओ।" \
+  --checkpoint model.safetensors --backend supabase --mcp --stats
+```
+
+The CLI automatically uses Apple Silicon MPS, CUDA or CPU and streams raw output. The bundled fictitious demo schema supports this example:
+
+```json
+{"action":"call","name":"execute_sql","arguments":{"query":"SELECT name FROM customers WHERE city = 'Delhi';"}}
+```
+
+Use `--context examples/context-supabase.json` or `examples/context-mysql.json`, edited with your schema and exact tool definitions. A context has `backend`, `project_id`, `schema`, `tools` and `policy`; tools use `name`, `description` and JSON Schema `inputSchema`. The context limit is 2,048 tokens including the output budget; final training sequences reached 945 tokens.
+
+`--mcp` validates and serializes a JSON-RPC `tools/call` payload. **It does not connect to an MCP server, authenticate, or execute a database query.** The host application must implement transport and execution. The structural validator does not prove semantic correctness. Invalid output causes a nonzero CLI exit status.
+
+A single local M5/MPS demonstration measured approximately 52 generated tokens/second, excluding model loading. This is one short run, not a throughput guarantee.
+
+## Evaluation
+
+Weights were frozen at step 25,715 on 2026-09-10 at 14:00:36 UTC, before reading test/manual results. Selection maximized the equal mean of full validation and development success among completed evaluated candidates. Development contains schema-layout and tool-name perturbations of validation families.
+
+| Measure | Held-out test | Additional manual phrasing |
+|---|---:|---:|
+| Examples | 1,196 | 160 |
+| Valid JSON | 1,196/1,196 (100%) | 160/160 (100%) |
+| Valid action/tool schema | 1,191/1,196 (99.58%) | 159/160 (99.38%) |
+| Correct tool, call cases | 1,088/1,100 (98.91%) | 146/152 (96.05%) |
+| Exact arguments, call cases | 1,014/1,100 (92.18%) | 124/152 (81.58%) |
+| SQL result equivalence, SQL cases | 896/912 (98.25%) | 94/112 (83.93%) |
+| **Full task success** | **1,106/1,196 (92.47%)** | **132/160 (82.50%)** |
+| TF-IDF retrieval + context-binding baseline, full success | 881/1,196 (73.66%) | 115/160 (71.88%) |
+
+These primary numbers are the completed CUDA BF16 evaluation captured in `evaluation/gpu-final-console-metrics.json`. The rented server subsequently refused SSH connections, so its raw final prediction files were not retrieved. Independent FP32 Mac evaluation reproduced every aggregate metric exactly; complete raw predictions, language/backend breakdowns and summaries are included in `evaluation/test-mac.*` and `evaluation/manual-mac.*`. No retraining or selection follows the test results.
+
+Evaluation uses raw greedy generation with no repair, constrained decoding, retrieval fallback or teacher fallback. Full SQL-task success requires the correct tool, exact non-SQL arguments (including project scope), compilation against the supplied schema, and matching results on two generated SQLite fixtures after dialect adaptation. SQL equivalence alone does not require correct project scope, so its percentage can exceed full success. Non-SQL tool calls require exact arguments; clarification/answer text requires exact reference wording.
+
+**These are synthetic benchmark results, not measured production task success.** Native MySQL 8.0.46/PostgreSQL 16.15 checks passed all 66 reference-operation/dialect cases; these validate reference data, not the final student's test predictions. Final native student evaluation was unavailable after SSH access failed. SQL fixture equivalence is not a proof for all possible databases.
+
+Validation scored 1,174/1,200 (97.83%); development scored 188/192 (97.92%). These were used for model selection and are not final held-out claims. The 160 manual cases use handwritten phrasing templates on 40 test families, so manual and test are not independent schema-family samples.
+
+## Architecture and training
+
+| Component | Value |
+|---|---|
+| Total parameters | 139,738,113 |
+| Decoder | 12 layers, width 1,024 |
+| Attention | 16 query heads, 4 KV heads, head dimension 64 |
+| Feed-forward | SwiGLU, intermediate width 2,816 |
+| Positions / normalization | RoPE, RMSNorm |
+| Embeddings | Tied input/output; train-only byte BPE vocabulary of 4,082 |
+| Source-copy head | Learned 128-dimensional pointer projection and mixture gate |
+| Auxiliary head | Three action classes, used in training |
+| Context | 2,048 |
+| Export | BF16 Safetensors, approximately 280 MB |
+
+The source-copy head mixes vocabulary generation with attention over the supplied context and question, excluding generated answer text. It adapts the established [pointer-generator approach](https://aclanthology.org/P17-1099/). The architecture and dataset evolved together, so this is not a controlled ablation or a claim of a novel research architecture. See [architecture documentation](docs/architecture.md).
+
+All student weights originate from random initialization during this experiment; the copy head was initialized later and trained with the continuing decoder. No pretrained student checkpoint or teacher logits were used. Training used BF16, AdamW, gradient clipping and weighted length buckets. Early loss weights were response 1, prompt 0.15, and auxiliary action CE 0.05. A later response-only refinement did not displace the selected checkpoint.
+
+The selected checkpoint records 595,067,301 processed tokens, 78,367,410 response tokens and 5,092.34 trainer seconds. These are retained-lineage counters, including validation/checkpoint time and excluding discarded work and model loading. Training continued to step 32,642, but those last weights were not selected. The overall experiment began at 10:57:23 UTC and had a 14:57:23 UTC deadline. See [experiment provenance](provenance/experiment-provenance.json) for curricula and counters.
+
+## Dataset and teacher
+
+The companion [TinyQuery-Tools-Multilingual dataset](https://huggingface.co/datasets/karmx/TinyQuery-Tools-Multilingual) contains 347,376 final training rows, 25,145 scenario groups, 100,224,712 tokens and 13,163,249 response tokens. Language/context variants are not independent teacher generations. SQL/tool semantics are programmatic; Qwen supplies templates, paraphrases and verification. An additional city-language curriculum is programmatically authored and labeled accordingly.
+
+Teacher: [Qwen/Qwen3.8-27B-FP8](https://huggingface.co/Qwen/Qwen3.8-27B-FP8), revision `017b9c7af6b5689d5dd426a76e0bc077eb5ca20a`. On the rented RTX PRO 6000, vLLM 0.28.0 with MTP-3 measured about 931 output tokens/second versus 691 without MTP at 32 concurrent requests. Initial template/paraphrase generation preceded MTP; verification and template audit used MTP. This is a measured workload comparison, not a claim of globally optimal settings. See [teacher runtime provenance](provenance/teacher-runtime.json) and [runtime recipes](runtime/README.md).
+
+Training has no scenario-group overlap with validation, development, test or manual. There are no exact prompt duplicates across splits. Development shares validation families, and manual shares test families, intentionally. A teacher audit flagged 37 template instances; 6,778 derived rows were removed from the final corpus after earlier training stages had already encountered them. Final reference-data audits passed schema/serialization and generated-fixture checks; these checks do not certify all natural-language paraphrases.
+
+## Scope and limitations
+
+- Supports a bounded recipe set: projections, filters, aggregates, grouping/HAVING, ordering, date/string filtering and a single join, plus schema discovery and a few generic tool categories.
+- Arbitrary unseen databases, tools, complex joins, long conversations and broad Hindi translation are not established capabilities. Hindi city/table lexical mappings cover a small explicit training vocabulary.
+- A correct JSON object can contain a wrong tool, literal, project ID or query. The model is not 100% reliable.
+- It emits one action, not a complete autonomous MCP agent. Database execution and authorization belong to the host application.
+- The runtime source and frozen data support new experiments; the release is not a bitwise replay of the evolving training session. The final optimizer state and some server logs were not recovered after SSH access failed.
+
+Weights and project code are released under Apache-2.0. Teacher provenance is documented separately. See `manifest.json` for file hashes and `checkpoint-info.json` for selected-weight counters.

checkpoint-info.json

@@ -0,0 +1 @@
+{"step": 25715, "processed_tokens": 595067301, "response_tokens": 78367410, "training_seconds": 5092.33953666687, "random_initialization": true}

config.json

@@ -0,0 +1,11 @@
+{
+  "vocab_size": 4082,
+  "width": 1024,
+  "layers": 12,
+  "heads": 16,
+  "kv_heads": 4,
+  "hidden": 2816,
+  "context": 2048,
+  "rope_theta": 10000.0,
+  "copy_dim": 128
+}

docs/architecture.md

@@ -0,0 +1,65 @@
+# TinyQuery: how the model works
+
+TinyQuery is a 139,738,113-parameter decoder trained from random weights for a narrow task: turn a question plus database schema and tool definitions into one JSON action. It uses established transformer components and a learned source-copy head. It is an experiment, not a claim of a new research architecture.
+
+```mermaid
+flowchart LR
+    A[Schema, tools and question] --> B[Byte BPE: 4,082 tokens]
+    B --> C[12 causal decoder blocks]
+    C --> D[Generate next-token probabilities]
+    C --> E[Copy probabilities over input tokens]
+    C --> F[Learned sigmoid gate]
+    D --> G[Mixed next-token distribution]
+    E --> G
+    F --> G
+    G --> H[Raw JSON action]
+    H --> I[Validate tool name and arguments]
+    I --> J[MCP tools/call payload]
+```
+
+The model sees everything needed for a particular request in its prompt. It must learn the language of the request, the required SQL operation, the correct runtime tool, and how to preserve identifiers and literal values. The inference CLI prints the action; it does not connect to a database or execute the tool.
+
+## Decoder
+
+| Component | Configuration |
+|---|---|
+| Tokenizer | Byte BPE trained only on training text; 4,082 tokens |
+| Hidden width | 1,024 |
+| Decoder layers | 12 |
+| Attention | 16 query heads, 4 key/value heads, head width 64 |
+| Positions | Rotary position embeddings, theta 10,000 |
+| Normalization | RMSNorm before attention and feed-forward layers |
+| Feed-forward | SwiGLU, intermediate width 2,816 |
+| Output projection | Tied to the input token embedding |
+| Context limit | 2,048 tokens; training coverage is shorter and reported with the dataset |
+| Training-only auxiliary head | Three action classes: call, clarify, answer |
+
+Attention is causal: a position can use earlier tokens but cannot see future answer tokens. Grouped-query attention shares four sets of keys and values across sixteen query heads, reducing the KV cache used during streaming. The cache preserves earlier attention keys and values so generation does not recompute the entire prompt for each token.
+
+## Learned copying
+
+Ordinary next-token generation initially memorized familiar table and tool names. Dataset variants and a copy head address this specific failure. The copy head projects decoder states into 128-dimensional queries and keys, attends to the supplied context and question, and adds together attention mass for repeated occurrences of the same token.
+
+For each next token, a learned gate combines two distributions:
+
+`P(token) = gate × P_generate(token) + (1 − gate) × P_copy_from_input(token)`
+
+The copy distribution excludes generated answer text. This prevents the model from repeatedly copying its own earlier mistakes. The gate and attention are learned; there is no hard-coded replacement of predicted tool names, no SQL template compiler, no constrained decoding, and no hidden teacher or retrieval fallback in neural evaluation.
+
+This adapts the established [pointer-generator idea](https://aclanthology.org/P17-1099/). Data and architecture changed together during development, so improvements are not presented as a controlled architecture ablation.
+
+## Training and output
+
+Early stages give answer tokens weight 1 and prompt tokens weight 0.15, plus 0.05 times an auxiliary action-classification loss. A late refinement uses prompt weight 0, focusing optimization on answers; final checkpoint selection can retain an earlier candidate if that refinement does not improve development results. The prompt contribution helps learn vocabulary from scratch; the answer contribution teaches the requested behavior. The final curriculum also samples particular context variants more often to counter identifier-format memorization.
+
+The teacher supplies language variations and a verification pass. Programmatic recipes supply reference SQL and tool actions. Student weights originate entirely from random initialization in this experiment; later stages continue those same weights and add randomly initialized copy parameters.
+
+The output contract is either a call, a clarification, or a short answer:
+
+```json
+{"action":"call","name":"execute_sql","arguments":{"query":"SELECT name FROM customers WHERE city = 'Delhi';"}}
+```
+
+The adapter checks the action against the supplied tool schema and can serialize a JSON-RPC `tools/call` request. JSON validity, correct tool selection, exact project scope and SQL result correctness are separate evaluation measures. A valid JSON action alone does not establish a correct query.
+
+The model's scope is bounded SQL and tool use in four language styles. It is not a general conversational model or a database knowledge store. Final measured results, training time and limitations belong in the release model card.

docs/experiment.md

@@ -0,0 +1,59 @@
+# TinyQuery Tools
+
+An experimental, randomly initialized decoder for schema-conditioned tool calls in English, imperfect English, Hindi and Hinglish. This is separate from the original 4.2M-parameter Tiny English learning project.
+
+The current student has **139,738,113 parameters**: 12 decoder layers, width 1024, 16 query heads, four KV heads, rotary positions, RMSNorm, SwiGLU width 2816, tied embeddings, a three-class auxiliary action head and a learned 128-dimensional source-copy head. The copy head mixes next-token generation with attention over the supplied context and question; it excludes generated answer text. The byte BPE vocabulary has 4,082 tokens learned from the training corpus. The configured context limit is 2,048; current training sequences reach 945 tokens. These are established transformer components with a custom training objective, not a demonstrated research breakthrough.
+
+## Output contract
+
+The model receives a backend, schema, runtime tool definitions and a question, and predicts one JSON action:
+
+```json
+{"action":"call","name":"execute_sql","arguments":{"query":"SELECT name FROM customers WHERE city = 'Delhi';"}}
+```
+
+Other actions are `{"action":"clarify","question":"..."}` and `{"action":"answer","text":"..."}`. Provided tool definitions use MCP-style `name`, `description`, and `inputSchema`. The adapter validates the action and can serialize a JSON-RPC `tools/call` request. **The neural model does not implement the MCP transport or authenticate to databases.**
+
+The database corpus targets read-only MySQL and PostgreSQL (Supabase). It covers projections, comparisons, NULLs, counts and aggregates, grouping/HAVING, sorting/limits, date filtering, substring matching and a single join. Schema discovery and bounded tool-error cases are included. Generic examples cover weather lookup, documentation search, file reading and ticket lookup. These are trained categories, not a claim of competence with arbitrary unseen tools.
+
+## Dataset
+
+The final continuation corpus contains 347,376 examples and 100,224,712 tokens, including 13,163,249 response tokens. It combines programmatic SQL/tool semantics, Qwen3.8-27B-FP8 multilingual templates and direct paraphrases, and randomized runtime identifiers. It has 25,145 scenario groups; rendered language/context variants are not independent teacher generations.
+
+Validation has 1,200 examples and test has 1,196, with held-out domain/table names and language templates. An additional 160 cases render hand-written phrasings on 40 test scenario families. These add phrasing coverage, not independent schema families. Neither test nor manual is used for training or model selection. A separate 192-case development split changes schema layouts and tool names on 140 validation families; it is used during development and shares no training/test/manual families.
+
+A teacher audit of 1,313 training SQL template instances flagged 37 variants; 6,778 derived rows were conservatively removed from the final corpus. Earlier training stages had already used them. Every final training action passed schema/serialization checks, and 151,920 distinct reference SQL cases executed on two generated SQLite fixtures. A separate 66-case reference check passed on native MySQL 8.0.46 and PostgreSQL 16.15. A further 160,253 distinct query/context pairs compile against the supplied schemas. These are reference-data checks, not student accuracy.
+
+Teacher revision: `017b9c7af6b5689d5dd426a76e0bc077eb5ca20a`. MTP-3 improved the measured 32-concurrent-request teacher workload from about 691 to 931 output tokens/second. Samples retain teacher prompts, settings and verification provenance. The model is trained from random weights; no pretrained student checkpoint or hidden teacher fallback is used.
+
+## Reproduce
+
+```sh
+python -m tinyquery.teacher_templates --out data/tinyquery/templates.jsonl
+python -m tinyquery.data --templates data/tinyquery/templates.jsonl --out data/tinyquery
+python -m tinyquery.prepare --data data/tinyquery
+python -m tinyquery.train --data data/tinyquery --out runs/tinyquery --copy-dim 128 --minutes 30
+```
+
+`concrete.py` and `verify_concrete.py` add direct paraphrases and round-trip verification. The published dataset includes the frozen augmented corpus; see its provenance and manifest rather than assuming the four commands above reproduce the exact published rows. An optional `--deadline` supplies a hard UTC wall-clock stop. The released experiment used multiple curriculum stages and a fixed four-hour overall budget; a new 30-minute run does not reproduce its result.
+
+## Stream output
+
+Once the finished export is placed in `runs/tinyquery/`:
+
+```sh
+.venv/bin/python -m tinyquery.chat "दिल्ली के ग्राहकों के नाम दिखाओ।" \
+  --checkpoint runs/tinyquery/model.safetensors --backend supabase --mcp
+```
+
+Use `--context path/to/context.json` to provide actual schema and tool definitions. The CLI streams the model's raw output and validates it. `--stats` reports measured generation speed. The MCP serializer rejects mismatched project scope, unknown schema tables and unsupported data-changing statements; these structural checks do not establish semantic correctness. It does not execute tools. The default context is a small fictitious customers table.
+
+## Evaluation
+
+```sh
+python -m tinyquery.evaluate --checkpoint runs/tinyquery/model.safetensors \
+  --tokenizer runs/tinyquery/tokenizer.json --data data/tinyquery/test.jsonl \
+  --out runs/tinyquery/test-predictions.jsonl
+```
+
+Evaluation uses raw greedy output with no JSON repair, constrained decoding or teacher fallback. Report JSON validity, tool-schema validity, tool choice, exact arguments and SQL result equivalence separately. The default SQL evaluator uses dialect adaptation to SQLite; `native_sql.py` performs separate checks on local MySQL/PostgreSQL engines. Final measured student results and failure examples will be written after training.

evaluation/baseline-manual.jsonl

@@ -0,0 +1,160 @@
+{"id": "manual_adff4eabe2639c0c128c_en", "retrieved_id": "a1b28dd3b5e89d70dccb_concrete_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_adff4eabe2639c0c128c_noisy_en", "retrieved_id": "755cf7dd59fa3c3634e3_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_adff4eabe2639c0c128c_hi", "retrieved_id": "b7f50817c901971c7f7a_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT description FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_adff4eabe2639c0c128c_hinglish", "retrieved_id": "a008cb35da6057a0324f_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE status = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_efee65268e6dc2bb069e_en", "retrieved_id": "b228aef51c67d0673184_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT customer_name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_efee65268e6dc2bb069e_noisy_en", "retrieved_id": "c91d2572f9472c9cec38_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT customer_name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_efee65268e6dc2bb069e_hi", "retrieved_id": "3aac2f1448f136869091_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT customer_name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_efee65268e6dc2bb069e_hinglish", "retrieved_id": "9f26d63bfba42c873223_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE rating > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_2c31add7d7b45843dec1_en", "retrieved_id": "6f49986bb93bef798e08_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM inspections WHERE kind = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_2c31add7d7b45843dec1_noisy_en", "retrieved_id": "53759e65346adaac1c7b_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM inspections WHERE kind = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_2c31add7d7b45843dec1_hi", "retrieved_id": "cbdf82266732007c41af_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM inspections WHERE kind = 'pending';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_2c31add7d7b45843dec1_hinglish", "retrieved_id": "6f49986bb93bef798e08_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM inspections WHERE kind = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_82c104614df1ce6e2762_en", "retrieved_id": "faee6951f2a9f841f1b7_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE salary > 100;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_82c104614df1ce6e2762_noisy_en", "retrieved_id": "944f318073a2bcd9624d_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE salary > 100;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_82c104614df1ce6e2762_hi", "retrieved_id": "7c6e0d36bbbdd7d0c41a_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE salary > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_82c104614df1ce6e2762_hinglish", "retrieved_id": "c9d4184209742377db82_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT DISTINCT category FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_35406fa024e66bcb5dd7_en", "retrieved_id": "93d7892cb1dbca43d963_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE score < 10000;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_35406fa024e66bcb5dd7_noisy_en", "retrieved_id": "d3e2b6e356dd98075c1f_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE score < 10000;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_35406fa024e66bcb5dd7_hi", "retrieved_id": "89b49bcc926a3e70ce83_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_35406fa024e66bcb5dd7_hinglish", "retrieved_id": "8bc1ddeb33b08c2be641_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE score >= 10000;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_0b8db53d711baed4f4ed_en", "retrieved_id": "ee309d02322a1ea31a21_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE cost > 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_0b8db53d711baed4f4ed_noisy_en", "retrieved_id": "3c390ab6e128eb92a927_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE cost >= 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_0b8db53d711baed4f4ed_hi", "retrieved_id": "5c402e1eb92eb1a39327_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE cost >= 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_0b8db53d711baed4f4ed_hinglish", "retrieved_id": "75239d67882146f98408_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE cost >= 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_b02f421828127e2eb96b_en", "retrieved_id": "b34987da74688618f914_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price <= 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_b02f421828127e2eb96b_noisy_en", "retrieved_id": "f410e7d2e9f171dbfc08_concrete_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM exhibits GROUP BY kind HAVING COUNT(*) > 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_b02f421828127e2eb96b_hi", "retrieved_id": "b34987da74688618f914_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price <= 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_b02f421828127e2eb96b_hinglish", "retrieved_id": "b706c1a087379cbbac6d_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price <= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_2e2ac3c5dcf95429daaa_en", "retrieved_id": "dbb34637fd547075f1d6_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_2e2ac3c5dcf95429daaa_noisy_en", "retrieved_id": "c8fc7444d05f02f427f0_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_2e2ac3c5dcf95429daaa_hi", "retrieved_id": "f7d9a89b39671e7655a1_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_2e2ac3c5dcf95429daaa_hinglish", "retrieved_id": "c8fc7444d05f02f427f0_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_40464b00088c9159e3d0_en", "retrieved_id": "1ba145f4ee3b2aa19fe8_en", "output": "{\"action\":\"call\",\"name\":\"app.get_weather\",\"arguments\":{\"city\":\"Mumbai\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": false, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": false, "success": false, "error": "Unknown tool"}}
+{"id": "manual_40464b00088c9159e3d0_noisy_en", "retrieved_id": "b339be824e371ca04217_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT SUM(rating) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_40464b00088c9159e3d0_hi", "retrieved_id": "36057c87ac77846cee49_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT COUNT(*) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_40464b00088c9159e3d0_hinglish", "retrieved_id": "6040c6883947ec5b3821_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM inspections;\",\"project_id\":\"demo_inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_d735c1ac1979ff1e8620_en", "retrieved_id": "caba0067fdd24c562152_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(price) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_d735c1ac1979ff1e8620_noisy_en", "retrieved_id": "caba0067fdd24c562152_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(price) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_d735c1ac1979ff1e8620_hi", "retrieved_id": "207b7dbc231f51af4760_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT kind, SUM(price) FROM inspections GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_d735c1ac1979ff1e8620_hinglish", "retrieved_id": "8703856ba26d23f81d4e_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(price) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_8efcf517aee64807a8e5_en", "retrieved_id": "97a28182b0b973aa9701_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_8efcf517aee64807a8e5_noisy_en", "retrieved_id": "b9936b4bad68a1e50b75_concrete_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_8efcf517aee64807a8e5_hi", "retrieved_id": "e16c4dae0f3afac7f310_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_8efcf517aee64807a8e5_hinglish", "retrieved_id": "75e3910b4be843450062_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_cab55a7429427fbc5093_en", "retrieved_id": "39cfbf14abe75381fce5_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_cab55a7429427fbc5093_noisy_en", "retrieved_id": "fbfc633a68d2a1ee4dc1_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_cab55a7429427fbc5093_hi", "retrieved_id": "fd60f3f12d6b58fc4c63_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_cab55a7429427fbc5093_hinglish", "retrieved_id": "851c98b607598265e5b5_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_b9c176c07809f42564a0_en", "retrieved_id": "89700fa6c178672acbb9_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT MAX(amount) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_b9c176c07809f42564a0_noisy_en", "retrieved_id": "72d5e4666d38219ea72f_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_b9c176c07809f42564a0_hi", "retrieved_id": "1ec2512105f780c64272_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY CASE WHEN amount IS NULL THEN 1 ELSE 0 END DESC, amount DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_b9c176c07809f42564a0_hinglish", "retrieved_id": "c81d8353510c1d7e35d5_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_475287613764159ee5eb_en", "retrieved_id": "27c83f55d9755a9c140a_concrete_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT category, SUM(amount) FROM reservations GROUP BY category;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_475287613764159ee5eb_noisy_en", "retrieved_id": "39ffd26f283fc2242233_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_475287613764159ee5eb_hi", "retrieved_id": "f5157d5860b54b580212_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT category FROM reservations;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_475287613764159ee5eb_hinglish", "retrieved_id": "6dfc3a46fb6c06bcfe9b_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_5a8f2e207d0efe5b018c_en", "retrieved_id": "9f9bcfa7a1b76cdf247b_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"],\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_5a8f2e207d0efe5b018c_noisy_en", "retrieved_id": "eebe3b000d91bc99d054_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"],\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_5a8f2e207d0efe5b018c_hi", "retrieved_id": "d862008c541c7c0edae1_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM specimens;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false}}
+{"id": "manual_5a8f2e207d0efe5b018c_hinglish", "retrieved_id": "fb4e17dc10585daed130_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"],\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_c4210296305105ab7472_en", "retrieved_id": "08dae4c9658c14af9a8b_noisy_en", "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
+{"id": "manual_c4210296305105ab7472_noisy_en", "retrieved_id": "981c50947db171d5fec5_hinglish", "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false, "text_exact": false}}
+{"id": "manual_c4210296305105ab7472_hi", "retrieved_id": "55ce86b87b3b3440fc8e_hi", "output": "{\"action\":\"clarify\",\"question\":\"सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
+{"id": "manual_c4210296305105ab7472_hinglish", "retrieved_id": "981c50947db171d5fec5_hinglish", "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
+{"id": "manual_754691bd1eea25d41ccf_en", "retrieved_id": "1173d51478c0f0770b25_en", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_754691bd1eea25d41ccf_noisy_en", "retrieved_id": "1173d51478c0f0770b25_en", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_754691bd1eea25d41ccf_hi", "retrieved_id": "04fd2c1a67c04de23381_hi", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_754691bd1eea25d41ccf_hinglish", "retrieved_id": "1173d51478c0f0770b25_en", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_252c6053295d69a93c94_en", "retrieved_id": "ba47da34289636eb8037_noisy_en", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_252c6053295d69a93c94_noisy_en", "retrieved_id": "3ef00bb2f51a6ed18daa_en", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_252c6053295d69a93c94_hi", "retrieved_id": "61c17ce29d5fca0cef4f_hi", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_252c6053295d69a93c94_hinglish", "retrieved_id": "750a046184db7426f758_noisy_en", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_f73e21e0eb10d6617995_en", "retrieved_id": "03bb9beb503dc3f084f0_en", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_f73e21e0eb10d6617995_noisy_en", "retrieved_id": "8705522869caaa48e039_noisy_en", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_f73e21e0eb10d6617995_hi", "retrieved_id": "8705522869caaa48e039_hi", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_f73e21e0eb10d6617995_hinglish", "retrieved_id": "0e78e9ded20fe5272477_hinglish", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_8a696e3c3522dd69d5ac_en", "retrieved_id": "8f99a3bc5f418131697b_noisy_en", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-1809\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
+{"id": "manual_8a696e3c3522dd69d5ac_noisy_en", "retrieved_id": "8730c789c1fa0e697d4f_hinglish", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-127\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
+{"id": "manual_8a696e3c3522dd69d5ac_hi", "retrieved_id": "city_29306ae64f1590153417_hi_layout_99707", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM contacts WHERE city = 'Delhi';\",\"project_id\":\"demo_inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false}}
+{"id": "manual_8a696e3c3522dd69d5ac_hinglish", "retrieved_id": "8730c789c1fa0e697d4f_hinglish", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-127\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
+{"id": "manual_2ad3e251c936135e2005_en", "retrieved_id": "a1b28dd3b5e89d70dccb_concrete_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_2ad3e251c936135e2005_noisy_en", "retrieved_id": "755cf7dd59fa3c3634e3_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_2ad3e251c936135e2005_hi", "retrieved_id": "b7f50817c901971c7f7a_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_2ad3e251c936135e2005_hinglish", "retrieved_id": "a008cb35da6057a0324f_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE kind = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_a060620c638d2dc3759e_en", "retrieved_id": "48b174ae126a25408b6f_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_a060620c638d2dc3759e_noisy_en", "retrieved_id": "739c908473cacc7eadc6_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_a060620c638d2dc3759e_hi", "retrieved_id": "04c518f7497c48a695a7_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_a060620c638d2dc3759e_hinglish", "retrieved_id": "9f26d63bfba42c873223_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE balance > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_eb982cf071094adaca4f_en", "retrieved_id": "f0fd5b687a47200d08cc_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT name FROM reservations WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_eb982cf071094adaca4f_noisy_en", "retrieved_id": "f0fd5b687a47200d08cc_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT name FROM reservations WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_eb982cf071094adaca4f_hi", "retrieved_id": "cbdf82266732007c41af_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM reservations WHERE category = 'pending';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_eb982cf071094adaca4f_hinglish", "retrieved_id": "f0fd5b687a47200d08cc_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT name FROM reservations WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_c023a5b6f95173ae856d_en", "retrieved_id": "cfd917d72e37aed08030_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE score > 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_c023a5b6f95173ae856d_noisy_en", "retrieved_id": "734a4347b6aef6b54122_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE score > 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_c023a5b6f95173ae856d_hi", "retrieved_id": "8de2a0fe75d4cb6dc3b8_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE score > 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_c023a5b6f95173ae856d_hinglish", "retrieved_id": "c9d4184209742377db82_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT DISTINCT kind FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_affa5e5e14444cd82bb4_en", "retrieved_id": "6a3bbef7eab5a98a8204_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE salary < 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_affa5e5e14444cd82bb4_noisy_en", "retrieved_id": "6a3bbef7eab5a98a8204_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE salary < 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_affa5e5e14444cd82bb4_hi", "retrieved_id": "78d0a9c46fad0d5b9731_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY salary ASC LIMIT 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_affa5e5e14444cd82bb4_hinglish", "retrieved_id": "7ab81eb1923cb8e1c5ea_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE salary < 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_c4f25575abfd24e3cd63_en", "retrieved_id": "44b7c6f0585479334d72_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE cost <= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_c4f25575abfd24e3cd63_noisy_en", "retrieved_id": "6c759f421afd3b87729c_concrete_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT region, COUNT(*) FROM reservations GROUP BY region HAVING COUNT(*) > 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_c4f25575abfd24e3cd63_hi", "retrieved_id": "c64c305ca0ce02c3d563_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE cost >= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_c4f25575abfd24e3cd63_hinglish", "retrieved_id": "b706c1a087379cbbac6d_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE cost <= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_23c0aa3a4596f995ad2a_en", "retrieved_id": "1ba145f4ee3b2aa19fe8_en", "output": "{\"action\":\"call\",\"name\":\"app.get_weather\",\"arguments\":{\"city\":\"Mumbai\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": false, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": false, "success": false, "error": "Unknown tool"}}
+{"id": "manual_23c0aa3a4596f995ad2a_noisy_en", "retrieved_id": "b339be824e371ca04217_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT SUM(amount) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_23c0aa3a4596f995ad2a_hi", "retrieved_id": "36057c87ac77846cee49_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_23c0aa3a4596f995ad2a_hinglish", "retrieved_id": "6040c6883947ec5b3821_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_b564e1a7b1e7fc6d2ab4_en", "retrieved_id": "8d7c829657a8468028fe_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(score) FROM specimens;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_b564e1a7b1e7fc6d2ab4_noisy_en", "retrieved_id": "e88c7ea3cba7722735db_concrete_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(score) FROM specimens;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_b564e1a7b1e7fc6d2ab4_hi", "retrieved_id": "c71e53cd4a9240c41192_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(score) FROM specimens;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_b564e1a7b1e7fc6d2ab4_hinglish", "retrieved_id": "6ba6fa33bb36ac7792b9_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(score) FROM specimens;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_2164b376f732a1c371ab_en", "retrieved_id": "27c83f55d9755a9c140a_concrete_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, SUM(score) FROM specimens GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_2164b376f732a1c371ab_noisy_en", "retrieved_id": "0517beaf82464fadc60f_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM specimens GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_2164b376f732a1c371ab_hi", "retrieved_id": "d367c82ffd966970c68f_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT DISTINCT kind FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_2164b376f732a1c371ab_hinglish", "retrieved_id": "3b1bddc042f483280893_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM specimens GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_4e42bbd40467b2b8c27f_en", "retrieved_id": "6821429a91693b8eafb4_noisy_en", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"SQL joins\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_4e42bbd40467b2b8c27f_noisy_en", "retrieved_id": "e1237888e640d85992e3_en", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"SQL joins\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_4e42bbd40467b2b8c27f_hi", "retrieved_id": "33f141708d95562cf305_hi", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"SQL joins\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_4e42bbd40467b2b8c27f_hinglish", "retrieved_id": "a336da01d4ae0dde0687_noisy_en", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"SQL joins\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_e8f90532d34c12209a83_en", "retrieved_id": "fe0c250a5b61d276b6ce_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE salary > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_e8f90532d34c12209a83_noisy_en", "retrieved_id": "980ddf0f65b7700a2fb6_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(salary) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_e8f90532d34c12209a83_hi", "retrieved_id": "df2bcb22f6685fc818a1_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE salary < 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_e8f90532d34c12209a83_hinglish", "retrieved_id": "f20130bbd19fc0c8fd2a_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE salary >= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_04b2e5886e750df09a51_en", "retrieved_id": "348951272c18ab797a78_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE display_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_04b2e5886e750df09a51_noisy_en", "retrieved_id": "c9ac52ed487c1889af4f_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE display_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_04b2e5886e750df09a51_hi", "retrieved_id": "f7d9a89b39671e7655a1_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE display_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_04b2e5886e750df09a51_hinglish", "retrieved_id": "0c268171e2f1d1321d9b_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE display_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_d30264bbc3b08aedbc51_en", "retrieved_id": "4ef6b6c0e983997e2ad0_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT SUM(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_d30264bbc3b08aedbc51_noisy_en", "retrieved_id": "4ef6b6c0e983997e2ad0_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT SUM(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_d30264bbc3b08aedbc51_hi", "retrieved_id": "207b7dbc231f51af4760_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT city, SUM(score) FROM exhibits GROUP BY city;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_d30264bbc3b08aedbc51_hinglish", "retrieved_id": "b36fba3845c7dc8f8585_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT SUM(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_7dd163238936d96d0bd4_en", "retrieved_id": "64b95cbab60bce765622_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT city FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_7dd163238936d96d0bd4_noisy_en", "retrieved_id": "ba18b57d6a751c27e75e_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT city FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_7dd163238936d96d0bd4_hi", "retrieved_id": "026f4d2a18eddb751c30_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT city FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_7dd163238936d96d0bd4_hinglish", "retrieved_id": "851c98b607598265e5b5_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT city FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_fd0bab1cc08d7298f131_en", "retrieved_id": "7cad2ba59a27227cbdd1_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(salary) FROM exhibits;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_fd0bab1cc08d7298f131_noisy_en", "retrieved_id": "331c202e8594d94e7dbc_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY salary ASC LIMIT 20;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_fd0bab1cc08d7298f131_hi", "retrieved_id": "1ec2512105f780c64272_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_fd0bab1cc08d7298f131_hinglish", "retrieved_id": "982a431f770356ef4ffe_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_6f3055c580b33340c854_en", "retrieved_id": "00319aa9372ffea135d7_en", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_6f3055c580b33340c854_noisy_en", "retrieved_id": "24837896562853f7b6a4_noisy_en", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_6f3055c580b33340c854_hi", "retrieved_id": "258c24044187ca1265c5_hi", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_6f3055c580b33340c854_hinglish", "retrieved_id": "d53f5c6036055adc1894_hinglish", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_5ebc52b3d20bde2076b3_en", "retrieved_id": "c745f3dc5489adf68488_en", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6124\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_5ebc52b3d20bde2076b3_noisy_en", "retrieved_id": "c745f3dc5489adf68488_en", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6124\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_5ebc52b3d20bde2076b3_hi", "retrieved_id": "c745f3dc5489adf68488_hi", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6124\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_5ebc52b3d20bde2076b3_hinglish", "retrieved_id": "c745f3dc5489adf68488_en", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6124\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_8ebc9e868ee558a8ddfe_en", "retrieved_id": "08dae4c9658c14af9a8b_noisy_en", "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
+{"id": "manual_8ebc9e868ee558a8ddfe_noisy_en", "retrieved_id": "981c50947db171d5fec5_hinglish", "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false, "text_exact": false}}
+{"id": "manual_8ebc9e868ee558a8ddfe_hi", "retrieved_id": "55ce86b87b3b3440fc8e_hi", "output": "{\"action\":\"clarify\",\"question\":\"सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
+{"id": "manual_8ebc9e868ee558a8ddfe_hinglish", "retrieved_id": "981c50947db171d5fec5_hinglish", "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
+{"id": "manual_88b478ce80eed61bdfb4_en", "retrieved_id": "03a2503aa4caeb5b0396_en", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_88b478ce80eed61bdfb4_noisy_en", "retrieved_id": "03a2503aa4caeb5b0396_en", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_88b478ce80eed61bdfb4_hi", "retrieved_id": "fcc2c16620594edcdeae_hi", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_88b478ce80eed61bdfb4_hinglish", "retrieved_id": "03a2503aa4caeb5b0396_en", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_9bf8948becf74e81a103_en", "retrieved_id": "9f9bcfa7a1b76cdf247b_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_9bf8948becf74e81a103_noisy_en", "retrieved_id": "eebe3b000d91bc99d054_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_9bf8948becf74e81a103_hi", "retrieved_id": "d862008c541c7c0edae1_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false}}
+{"id": "manual_9bf8948becf74e81a103_hinglish", "retrieved_id": "fb4e17dc10585daed130_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}

evaluation/baseline-manual.summary.json

@@ -0,0 +1,1007 @@
+{
+  "baseline": "TF-IDF nearest question plus tool-description and DDL-position binding; no expected answer or gold slots used",
+  "training_questions": 212073,
+  "examples": 160,
+  "seconds": 7.722788095474243,
+  "metrics": {
+    "json_valid": {
+      "correct": 160,
+      "total": 160,
+      "rate": 1.0
+    },
+    "schema_valid": {
+      "correct": 158,
+      "total": 160,
+      "rate": 0.9875
+    },
+    "action_correct": {
+      "correct": 160,
+      "total": 160,
+      "rate": 1.0
+    },
+    "tool_correct": {
+      "correct": 147,
+      "total": 152,
+      "rate": 0.9671052631578947
+    },
+    "arguments_exact": {
+      "correct": 109,
+      "total": 152,
+      "rate": 0.7171052631578947
+    },
+    "sql_equivalent": {
+      "correct": 75,
+      "total": 112,
+      "rate": 0.6696428571428571
+    },
+    "success": {
+      "correct": 115,
+      "total": 160,
+      "rate": 0.71875
+    }
+  },
+  "groups": {
+    "language:en": {
+      "json_valid": {
+        "correct": 40,
+        "total": 40,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 38,
+        "total": 40,
+        "rate": 0.95
+      },
+      "action_correct": {
+        "correct": 40,
+        "total": 40,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 36,
+        "total": 38,
+        "rate": 0.9473684210526315
+      },
+      "arguments_exact": {
+        "correct": 29,
+        "total": 38,
+        "rate": 0.7631578947368421
+      },
+      "sql_equivalent": {
+        "correct": 20,
+        "total": 28,
+        "rate": 0.7142857142857143
+      },
+      "success": {
+        "correct": 31,
+        "total": 40,
+        "rate": 0.775
+      }
+    },
+    "backend:mysql": {
+      "json_valid": {
+        "correct": 80,
+        "total": 80,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 79,
+        "total": 80,
+        "rate": 0.9875
+      },
+      "action_correct": {
+        "correct": 80,
+        "total": 80,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 74,
+        "total": 76,
+        "rate": 0.9736842105263158
+      },
+      "arguments_exact": {
+        "correct": 57,
+        "total": 76,
+        "rate": 0.75
+      },
+      "sql_equivalent": {
+        "correct": 38,
+        "total": 56,
+        "rate": 0.6785714285714286
+      },
+      "success": {
+        "correct": 60,
+        "total": 80,
+        "rate": 0.75
+      }
+    },
+    "operation:all": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 4,
+        "total": 8,
+        "rate": 0.5
+      },
+      "sql_equivalent": {
+        "correct": 4,
+        "total": 8,
+        "rate": 0.5
+      },
+      "success": {
+        "correct": 4,
+        "total": 8,
+        "rate": 0.5
+      }
+    },
+    "language:noisy_en": {
+      "json_valid": {
+        "correct": 40,
+        "total": 40,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 40,
+        "total": 40,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 40,
+        "total": 40,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 38,
+        "total": 38,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 31,
+        "total": 38,
+        "rate": 0.8157894736842105
+      },
+      "sql_equivalent": {
+        "correct": 22,
+        "total": 28,
+        "rate": 0.7857142857142857
+      },
+      "success": {
+        "correct": 31,
+        "total": 40,
+        "rate": 0.775
+      }
+    },
+    "language:hi": {
+      "json_valid": {
+        "correct": 40,
+        "total": 40,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 40,
+        "total": 40,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 40,
+        "total": 40,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 35,
+        "total": 38,
+        "rate": 0.9210526315789473
+      },
+      "arguments_exact": {
+        "correct": 21,
+        "total": 38,
+        "rate": 0.5526315789473685
+      },
+      "sql_equivalent": {
+        "correct": 14,
+        "total": 28,
+        "rate": 0.5
+      },
+      "success": {
+        "correct": 23,
+        "total": 40,
+        "rate": 0.575
+      }
+    },
+    "language:hinglish": {
+      "json_valid": {
+        "correct": 40,
+        "total": 40,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 40,
+        "total": 40,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 40,
+        "total": 40,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 38,
+        "total": 38,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 38,
+        "rate": 0.7368421052631579
+      },
+      "sql_equivalent": {
+        "correct": 19,
+        "total": 28,
+        "rate": 0.6785714285714286
+      },
+      "success": {
+        "correct": 30,
+        "total": 40,
+        "rate": 0.75
+      }
+    },
+    "backend:supabase": {
+      "json_valid": {
+        "correct": 80,
+        "total": 80,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 79,
+        "total": 80,
+        "rate": 0.9875
+      },
+      "action_correct": {
+        "correct": 80,
+        "total": 80,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 73,
+        "total": 76,
+        "rate": 0.9605263157894737
+      },
+      "arguments_exact": {
+        "correct": 52,
+        "total": 76,
+        "rate": 0.6842105263157895
+      },
+      "sql_equivalent": {
+        "correct": 37,
+        "total": 56,
+        "rate": 0.6607142857142857
+      },
+      "success": {
+        "correct": 55,
+        "total": 80,
+        "rate": 0.6875
+      }
+    },
+    "operation:project": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 6,
+        "total": 8,
+        "rate": 0.75
+      },
+      "sql_equivalent": {
+        "correct": 6,
+        "total": 8,
+        "rate": 0.75
+      },
+      "success": {
+        "correct": 6,
+        "total": 8,
+        "rate": 0.75
+      }
+    },
+    "operation:project_eq": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 6,
+        "total": 8,
+        "rate": 0.75
+      },
+      "sql_equivalent": {
+        "correct": 6,
+        "total": 8,
+        "rate": 0.75
+      },
+      "success": {
+        "correct": 6,
+        "total": 8,
+        "rate": 0.75
+      }
+    },
+    "operation:gt": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 6,
+        "total": 8,
+        "rate": 0.75
+      },
+      "sql_equivalent": {
+        "correct": 6,
+        "total": 8,
+        "rate": 0.75
+      },
+      "success": {
+        "correct": 6,
+        "total": 8,
+        "rate": 0.75
+      }
+    },
+    "operation:lt": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 5,
+        "total": 8,
+        "rate": 0.625
+      },
+      "sql_equivalent": {
+        "correct": 5,
+        "total": 8,
+        "rate": 0.625
+      },
+      "success": {
+        "correct": 5,
+        "total": 8,
+        "rate": 0.625
+      }
+    },
+    "operation:gte": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 4,
+        "total": 8,
+        "rate": 0.5
+      },
+      "sql_equivalent": {
+        "correct": 4,
+        "total": 8,
+        "rate": 0.5
+      },
+      "success": {
+        "correct": 4,
+        "total": 8,
+        "rate": 0.5
+      }
+    },
+    "operation:lte": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 4,
+        "total": 8,
+        "rate": 0.5
+      },
+      "sql_equivalent": {
+        "correct": 4,
+        "total": 8,
+        "rate": 0.5
+      },
+      "success": {
+        "correct": 4,
+        "total": 8,
+        "rate": 0.5
+      }
+    },
+    "operation:null": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      }
+    },
+    "operation:count": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 6,
+        "total": 8,
+        "rate": 0.75
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 6,
+        "total": 8,
+        "rate": 0.75
+      },
+      "arguments_exact": {
+        "correct": 4,
+        "total": 8,
+        "rate": 0.5
+      },
+      "sql_equivalent": {
+        "correct": 4,
+        "total": 8,
+        "rate": 0.5
+      },
+      "success": {
+        "correct": 4,
+        "total": 8,
+        "rate": 0.5
+      }
+    },
+    "operation:sum": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 5,
+        "total": 8,
+        "rate": 0.625
+      },
+      "sql_equivalent": {
+        "correct": 5,
+        "total": 8,
+        "rate": 0.625
+      },
+      "success": {
+        "correct": 5,
+        "total": 8,
+        "rate": 0.625
+      }
+    },
+    "operation:avg": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      }
+    },
+    "operation:distinct": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      }
+    },
+    "operation:top": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 3,
+        "total": 8,
+        "rate": 0.375
+      },
+      "sql_equivalent": {
+        "correct": 3,
+        "total": 8,
+        "rate": 0.375
+      },
+      "success": {
+        "correct": 3,
+        "total": 8,
+        "rate": 0.375
+      }
+    },
+    "operation:group_count": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 4,
+        "total": 8,
+        "rate": 0.5
+      },
+      "sql_equivalent": {
+        "correct": 4,
+        "total": 8,
+        "rate": 0.5
+      },
+      "success": {
+        "correct": 4,
+        "total": 8,
+        "rate": 0.5
+      }
+    },
+    "operation:list_tables": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 6,
+        "total": 8,
+        "rate": 0.75
+      },
+      "arguments_exact": {
+        "correct": 6,
+        "total": 8,
+        "rate": 0.75
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 6,
+        "total": 8,
+        "rate": 0.75
+      }
+    },
+    "operation:ambiguous": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "arguments_exact": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 6,
+        "total": 8,
+        "rate": 0.75
+      }
+    },
+    "operation:weather": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      }
+    },
+    "operation:search": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      }
+    },
+    "operation:read_file": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      }
+    },
+    "operation:ticket": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 7,
+        "total": 8,
+        "rate": 0.875
+      },
+      "arguments_exact": {
+        "correct": 4,
+        "total": 8,
+        "rate": 0.5
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 4,
+        "total": 8,
+        "rate": 0.5
+      }
+    }
+  }
+}

evaluation/baseline-test.summary.json

@@ -0,0 +1,1932 @@
+{
+  "baseline": "TF-IDF nearest question plus tool-description and DDL-position binding; no expected answer or gold slots used",
+  "training_questions": 212073,
+  "examples": 1196,
+  "seconds": 9.818428039550781,
+  "metrics": {
+    "json_valid": {
+      "correct": 1196,
+      "total": 1196,
+      "rate": 1.0
+    },
+    "schema_valid": {
+      "correct": 1187,
+      "total": 1196,
+      "rate": 0.9924749163879598
+    },
+    "action_correct": {
+      "correct": 1184,
+      "total": 1196,
+      "rate": 0.9899665551839465
+    },
+    "tool_correct": {
+      "correct": 1077,
+      "total": 1100,
+      "rate": 0.9790909090909091
+    },
+    "arguments_exact": {
+      "correct": 753,
+      "total": 1100,
+      "rate": 0.6845454545454546
+    },
+    "sql_equivalent": {
+      "correct": 664,
+      "total": 912,
+      "rate": 0.7280701754385965
+    },
+    "success": {
+      "correct": 881,
+      "total": 1196,
+      "rate": 0.7366220735785953
+    }
+  },
+  "groups": {
+    "language:en": {
+      "json_valid": {
+        "correct": 299,
+        "total": 299,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 296,
+        "total": 299,
+        "rate": 0.9899665551839465
+      },
+      "action_correct": {
+        "correct": 293,
+        "total": 299,
+        "rate": 0.979933110367893
+      },
+      "tool_correct": {
+        "correct": 268,
+        "total": 275,
+        "rate": 0.9745454545454545
+      },
+      "arguments_exact": {
+        "correct": 169,
+        "total": 275,
+        "rate": 0.6145454545454545
+      },
+      "sql_equivalent": {
+        "correct": 146,
+        "total": 228,
+        "rate": 0.6403508771929824
+      },
+      "success": {
+        "correct": 197,
+        "total": 299,
+        "rate": 0.6588628762541806
+      }
+    },
+    "backend:mysql": {
+      "json_valid": {
+        "correct": 608,
+        "total": 608,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 608,
+        "total": 608,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 601,
+        "total": 608,
+        "rate": 0.9884868421052632
+      },
+      "tool_correct": {
+        "correct": 543,
+        "total": 556,
+        "rate": 0.9766187050359713
+      },
+      "arguments_exact": {
+        "correct": 391,
+        "total": 556,
+        "rate": 0.7032374100719424
+      },
+      "sql_equivalent": {
+        "correct": 351,
+        "total": 476,
+        "rate": 0.7373949579831933
+      },
+      "success": {
+        "correct": 452,
+        "total": 608,
+        "rate": 0.743421052631579
+      }
+    },
+    "operation:all": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "language:noisy_en": {
+      "json_valid": {
+        "correct": 299,
+        "total": 299,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 299,
+        "total": 299,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 293,
+        "total": 299,
+        "rate": 0.979933110367893
+      },
+      "tool_correct": {
+        "correct": 270,
+        "total": 275,
+        "rate": 0.9818181818181818
+      },
+      "arguments_exact": {
+        "correct": 195,
+        "total": 275,
+        "rate": 0.7090909090909091
+      },
+      "sql_equivalent": {
+        "correct": 166,
+        "total": 228,
+        "rate": 0.7280701754385965
+      },
+      "success": {
+        "correct": 221,
+        "total": 299,
+        "rate": 0.7391304347826086
+      }
+    },
+    "language:hi": {
+      "json_valid": {
+        "correct": 299,
+        "total": 299,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 299,
+        "total": 299,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 299,
+        "total": 299,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 270,
+        "total": 275,
+        "rate": 0.9818181818181818
+      },
+      "arguments_exact": {
+        "correct": 183,
+        "total": 275,
+        "rate": 0.6654545454545454
+      },
+      "sql_equivalent": {
+        "correct": 162,
+        "total": 228,
+        "rate": 0.7105263157894737
+      },
+      "success": {
+        "correct": 222,
+        "total": 299,
+        "rate": 0.7424749163879598
+      }
+    },
+    "language:hinglish": {
+      "json_valid": {
+        "correct": 299,
+        "total": 299,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 293,
+        "total": 299,
+        "rate": 0.979933110367893
+      },
+      "action_correct": {
+        "correct": 299,
+        "total": 299,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 269,
+        "total": 275,
+        "rate": 0.9781818181818182
+      },
+      "arguments_exact": {
+        "correct": 206,
+        "total": 275,
+        "rate": 0.7490909090909091
+      },
+      "sql_equivalent": {
+        "correct": 190,
+        "total": 228,
+        "rate": 0.8333333333333334
+      },
+      "success": {
+        "correct": 241,
+        "total": 299,
+        "rate": 0.8060200668896321
+      }
+    },
+    "backend:supabase": {
+      "json_valid": {
+        "correct": 588,
+        "total": 588,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 579,
+        "total": 588,
+        "rate": 0.9846938775510204
+      },
+      "action_correct": {
+        "correct": 583,
+        "total": 588,
+        "rate": 0.9914965986394558
+      },
+      "tool_correct": {
+        "correct": 534,
+        "total": 544,
+        "rate": 0.9816176470588235
+      },
+      "arguments_exact": {
+        "correct": 362,
+        "total": 544,
+        "rate": 0.6654411764705882
+      },
+      "sql_equivalent": {
+        "correct": 313,
+        "total": 436,
+        "rate": 0.7178899082568807
+      },
+      "success": {
+        "correct": 429,
+        "total": 588,
+        "rate": 0.7295918367346939
+      }
+    },
+    "operation:project": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 21,
+        "total": 28,
+        "rate": 0.75
+      },
+      "sql_equivalent": {
+        "correct": 21,
+        "total": 28,
+        "rate": 0.75
+      },
+      "success": {
+        "correct": 21,
+        "total": 28,
+        "rate": 0.75
+      }
+    },
+    "operation:eq": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 13,
+        "total": 28,
+        "rate": 0.4642857142857143
+      },
+      "sql_equivalent": {
+        "correct": 14,
+        "total": 28,
+        "rate": 0.5
+      },
+      "success": {
+        "correct": 14,
+        "total": 28,
+        "rate": 0.5
+      }
+    },
+    "operation:project_eq": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 26,
+        "total": 28,
+        "rate": 0.9285714285714286
+      },
+      "sql_equivalent": {
+        "correct": 26,
+        "total": 28,
+        "rate": 0.9285714285714286
+      },
+      "success": {
+        "correct": 26,
+        "total": 28,
+        "rate": 0.9285714285714286
+      }
+    },
+    "operation:gt": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 3,
+        "total": 28,
+        "rate": 0.10714285714285714
+      },
+      "sql_equivalent": {
+        "correct": 3,
+        "total": 28,
+        "rate": 0.10714285714285714
+      },
+      "success": {
+        "correct": 3,
+        "total": 28,
+        "rate": 0.10714285714285714
+      }
+    },
+    "operation:lt": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 27,
+        "total": 28,
+        "rate": 0.9642857142857143
+      },
+      "sql_equivalent": {
+        "correct": 27,
+        "total": 28,
+        "rate": 0.9642857142857143
+      },
+      "success": {
+        "correct": 27,
+        "total": 28,
+        "rate": 0.9642857142857143
+      }
+    },
+    "operation:gte": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 11,
+        "total": 28,
+        "rate": 0.39285714285714285
+      },
+      "sql_equivalent": {
+        "correct": 11,
+        "total": 28,
+        "rate": 0.39285714285714285
+      },
+      "success": {
+        "correct": 11,
+        "total": 28,
+        "rate": 0.39285714285714285
+      }
+    },
+    "operation:lte": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 13,
+        "total": 28,
+        "rate": 0.4642857142857143
+      },
+      "sql_equivalent": {
+        "correct": 13,
+        "total": 28,
+        "rate": 0.4642857142857143
+      },
+      "success": {
+        "correct": 13,
+        "total": 28,
+        "rate": 0.4642857142857143
+      }
+    },
+    "operation:between": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 24,
+        "total": 28,
+        "rate": 0.8571428571428571
+      },
+      "sql_equivalent": {
+        "correct": 24,
+        "total": 28,
+        "rate": 0.8571428571428571
+      },
+      "success": {
+        "correct": 24,
+        "total": 28,
+        "rate": 0.8571428571428571
+      }
+    },
+    "operation:and": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 16,
+        "total": 28,
+        "rate": 0.5714285714285714
+      },
+      "sql_equivalent": {
+        "correct": 16,
+        "total": 28,
+        "rate": 0.5714285714285714
+      },
+      "success": {
+        "correct": 16,
+        "total": 28,
+        "rate": 0.5714285714285714
+      }
+    },
+    "operation:or": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 13,
+        "total": 28,
+        "rate": 0.4642857142857143
+      },
+      "sql_equivalent": {
+        "correct": 23,
+        "total": 28,
+        "rate": 0.8214285714285714
+      },
+      "success": {
+        "correct": 23,
+        "total": 28,
+        "rate": 0.8214285714285714
+      }
+    },
+    "operation:null": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 18,
+        "total": 28,
+        "rate": 0.6428571428571429
+      },
+      "sql_equivalent": {
+        "correct": 18,
+        "total": 28,
+        "rate": 0.6428571428571429
+      },
+      "success": {
+        "correct": 18,
+        "total": 28,
+        "rate": 0.6428571428571429
+      }
+    },
+    "operation:not_null": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 14,
+        "total": 28,
+        "rate": 0.5
+      },
+      "sql_equivalent": {
+        "correct": 21,
+        "total": 28,
+        "rate": 0.75
+      },
+      "success": {
+        "correct": 21,
+        "total": 28,
+        "rate": 0.75
+      }
+    },
+    "operation:count": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 21,
+        "total": 28,
+        "rate": 0.75
+      },
+      "sql_equivalent": {
+        "correct": 21,
+        "total": 28,
+        "rate": 0.75
+      },
+      "success": {
+        "correct": 21,
+        "total": 28,
+        "rate": 0.75
+      }
+    },
+    "operation:count_eq": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 13,
+        "total": 28,
+        "rate": 0.4642857142857143
+      },
+      "sql_equivalent": {
+        "correct": 13,
+        "total": 28,
+        "rate": 0.4642857142857143
+      },
+      "success": {
+        "correct": 13,
+        "total": 28,
+        "rate": 0.4642857142857143
+      }
+    },
+    "operation:sum": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 27,
+        "total": 28,
+        "rate": 0.9642857142857143
+      },
+      "arguments_exact": {
+        "correct": 21,
+        "total": 28,
+        "rate": 0.75
+      },
+      "sql_equivalent": {
+        "correct": 21,
+        "total": 28,
+        "rate": 0.75
+      },
+      "success": {
+        "correct": 21,
+        "total": 28,
+        "rate": 0.75
+      }
+    },
+    "operation:avg": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:max": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 16,
+        "total": 28,
+        "rate": 0.5714285714285714
+      },
+      "sql_equivalent": {
+        "correct": 16,
+        "total": 28,
+        "rate": 0.5714285714285714
+      },
+      "success": {
+        "correct": 16,
+        "total": 28,
+        "rate": 0.5714285714285714
+      }
+    },
+    "operation:min": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:distinct": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 21,
+        "total": 28,
+        "rate": 0.75
+      },
+      "sql_equivalent": {
+        "correct": 21,
+        "total": 28,
+        "rate": 0.75
+      },
+      "success": {
+        "correct": 21,
+        "total": 28,
+        "rate": 0.75
+      }
+    },
+    "operation:sort_desc": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 12,
+        "total": 28,
+        "rate": 0.42857142857142855
+      },
+      "sql_equivalent": {
+        "correct": 12,
+        "total": 28,
+        "rate": 0.42857142857142855
+      },
+      "success": {
+        "correct": 12,
+        "total": 28,
+        "rate": 0.42857142857142855
+      }
+    },
+    "operation:sort_asc": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 12,
+        "total": 28,
+        "rate": 0.42857142857142855
+      },
+      "sql_equivalent": {
+        "correct": 12,
+        "total": 28,
+        "rate": 0.42857142857142855
+      },
+      "success": {
+        "correct": 12,
+        "total": 28,
+        "rate": 0.42857142857142855
+      }
+    },
+    "operation:top": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 8,
+        "total": 28,
+        "rate": 0.2857142857142857
+      },
+      "sql_equivalent": {
+        "correct": 8,
+        "total": 28,
+        "rate": 0.2857142857142857
+      },
+      "success": {
+        "correct": 8,
+        "total": 28,
+        "rate": 0.2857142857142857
+      }
+    },
+    "operation:bottom": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 17,
+        "total": 28,
+        "rate": 0.6071428571428571
+      },
+      "sql_equivalent": {
+        "correct": 17,
+        "total": 28,
+        "rate": 0.6071428571428571
+      },
+      "success": {
+        "correct": 17,
+        "total": 28,
+        "rate": 0.6071428571428571
+      }
+    },
+    "operation:group_count": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:group_sum": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:having": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 25,
+        "total": 28,
+        "rate": 0.8928571428571429
+      },
+      "sql_equivalent": {
+        "correct": 25,
+        "total": 28,
+        "rate": 0.8928571428571429
+      },
+      "success": {
+        "correct": 25,
+        "total": 28,
+        "rate": 0.8928571428571429
+      }
+    },
+    "operation:date_after": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 19,
+        "total": 28,
+        "rate": 0.6785714285714286
+      },
+      "sql_equivalent": {
+        "correct": 22,
+        "total": 28,
+        "rate": 0.7857142857142857
+      },
+      "success": {
+        "correct": 22,
+        "total": 28,
+        "rate": 0.7857142857142857
+      }
+    },
+    "operation:year": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 13,
+        "total": 28,
+        "rate": 0.4642857142857143
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:month": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 12,
+        "total": 28,
+        "rate": 0.42857142857142855
+      },
+      "sql_equivalent": {
+        "correct": 26,
+        "total": 28,
+        "rate": 0.9285714285714286
+      },
+      "success": {
+        "correct": 26,
+        "total": 28,
+        "rate": 0.9285714285714286
+      }
+    },
+    "operation:contains": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 19,
+        "total": 24,
+        "rate": 0.7916666666666666
+      },
+      "sql_equivalent": {
+        "correct": 19,
+        "total": 24,
+        "rate": 0.7916666666666666
+      },
+      "success": {
+        "correct": 19,
+        "total": 24,
+        "rate": 0.7916666666666666
+      }
+    },
+    "operation:join": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      }
+    },
+    "operation:join_filter": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 22,
+        "total": 24,
+        "rate": 0.9166666666666666
+      },
+      "sql_equivalent": {
+        "correct": 22,
+        "total": 24,
+        "rate": 0.9166666666666666
+      },
+      "success": {
+        "correct": 22,
+        "total": 24,
+        "rate": 0.9166666666666666
+      }
+    },
+    "operation:list_tables": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      }
+    },
+    "operation:describe": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 21,
+        "total": 24,
+        "rate": 0.875
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 12,
+        "total": 24,
+        "rate": 0.5
+      },
+      "arguments_exact": {
+        "correct": 9,
+        "total": 24,
+        "rate": 0.375
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 9,
+        "total": 24,
+        "rate": 0.375
+      }
+    },
+    "operation:missing_schema": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      }
+    },
+    "operation:sql_error": {
+      "json_valid": {
+        "correct": 20,
+        "total": 20,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 14,
+        "total": 20,
+        "rate": 0.7
+      },
+      "action_correct": {
+        "correct": 20,
+        "total": 20,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 10,
+        "total": 20,
+        "rate": 0.5
+      },
+      "arguments_exact": {
+        "correct": 6,
+        "total": 20,
+        "rate": 0.3
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 6,
+        "total": 20,
+        "rate": 0.3
+      }
+    },
+    "operation:ambiguous": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 18,
+        "total": 24,
+        "rate": 0.75
+      },
+      "tool_correct": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "arguments_exact": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 18,
+        "total": 24,
+        "rate": 0.75
+      }
+    },
+    "operation:missing_value": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "arguments_exact": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      }
+    },
+    "operation:write": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 18,
+        "total": 24,
+        "rate": 0.75
+      },
+      "tool_correct": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "arguments_exact": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 18,
+        "total": 24,
+        "rate": 0.75
+      }
+    },
+    "operation:weather": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      }
+    },
+    "operation:search": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      }
+    },
+    "operation:read_file": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      }
+    },
+    "operation:ticket": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 4,
+        "total": 24,
+        "rate": 0.16666666666666666
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 4,
+        "total": 24,
+        "rate": 0.16666666666666666
+      }
+    },
+    "operation:final": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "arguments_exact": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 18,
+        "total": 24,
+        "rate": 0.75
+      }
+    }
+  }
+}

evaluation/development.summary.json

@@ -0,0 +1,1785 @@
+{
+  "examples": 192,
+  "seconds": 6.547292470932007,
+  "decoding": "raw greedy, no repair, no teacher fallback",
+  "checkpoint_sha256": "9489b5cde69c11f64b3e7031182e8248aa83790f1ad4aeefbf99a4e5e152e692",
+  "sql_metric": "Dialect parsing and equivalent results on two generated SQLite fixtures after dialect adaptation; not native MySQL/PostgreSQL execution.",
+  "metrics": {
+    "json_valid": {
+      "correct": 192,
+      "total": 192,
+      "rate": 1.0
+    },
+    "schema_valid": {
+      "correct": 192,
+      "total": 192,
+      "rate": 1.0
+    },
+    "action_correct": {
+      "correct": 192,
+      "total": 192,
+      "rate": 1.0
+    },
+    "tool_correct": {
+      "correct": 192,
+      "total": 192,
+      "rate": 1.0
+    },
+    "arguments_exact": {
+      "correct": 188,
+      "total": 192,
+      "rate": 0.9791666666666666
+    },
+    "sql_equivalent": {
+      "correct": 125,
+      "total": 128,
+      "rate": 0.9765625
+    },
+    "success": {
+      "correct": 188,
+      "total": 192,
+      "rate": 0.9791666666666666
+    }
+  },
+  "groups": {
+    "language:noisy_en": {
+      "json_valid": {
+        "correct": 48,
+        "total": 48,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 48,
+        "total": 48,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 48,
+        "total": 48,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 48,
+        "total": 48,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 48,
+        "total": 48,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 31,
+        "total": 31,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 48,
+        "total": 48,
+        "rate": 1.0
+      }
+    },
+    "backend:supabase": {
+      "json_valid": {
+        "correct": 92,
+        "total": 92,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 92,
+        "total": 92,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 92,
+        "total": 92,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 92,
+        "total": 92,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 90,
+        "total": 92,
+        "rate": 0.9782608695652174
+      },
+      "sql_equivalent": {
+        "correct": 64,
+        "total": 66,
+        "rate": 0.9696969696969697
+      },
+      "success": {
+        "correct": 90,
+        "total": 92,
+        "rate": 0.9782608695652174
+      }
+    },
+    "operation:missing_schema": {
+      "json_valid": {
+        "correct": 7,
+        "total": 7,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 7,
+        "total": 7,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 7,
+        "total": 7,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 7,
+        "total": 7,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 7,
+        "total": 7,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 7,
+        "total": 7,
+        "rate": 1.0
+      }
+    },
+    "operation:project": {
+      "json_valid": {
+        "correct": 7,
+        "total": 7,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 7,
+        "total": 7,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 7,
+        "total": 7,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 7,
+        "total": 7,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 6,
+        "total": 7,
+        "rate": 0.8571428571428571
+      },
+      "sql_equivalent": {
+        "correct": 6,
+        "total": 7,
+        "rate": 0.8571428571428571
+      },
+      "success": {
+        "correct": 6,
+        "total": 7,
+        "rate": 0.8571428571428571
+      }
+    },
+    "language:hinglish": {
+      "json_valid": {
+        "correct": 52,
+        "total": 52,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 52,
+        "total": 52,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 52,
+        "total": 52,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 52,
+        "total": 52,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 51,
+        "total": 52,
+        "rate": 0.9807692307692307
+      },
+      "sql_equivalent": {
+        "correct": 36,
+        "total": 37,
+        "rate": 0.972972972972973
+      },
+      "success": {
+        "correct": 51,
+        "total": 52,
+        "rate": 0.9807692307692307
+      }
+    },
+    "operation:count_eq": {
+      "json_valid": {
+        "correct": 5,
+        "total": 5,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 5,
+        "total": 5,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 5,
+        "total": 5,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 5,
+        "total": 5,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 5,
+        "total": 5,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 5,
+        "total": 5,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 5,
+        "total": 5,
+        "rate": 1.0
+      }
+    },
+    "language:en": {
+      "json_valid": {
+        "correct": 44,
+        "total": 44,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 44,
+        "total": 44,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 44,
+        "total": 44,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 44,
+        "total": 44,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 42,
+        "total": 44,
+        "rate": 0.9545454545454546
+      },
+      "sql_equivalent": {
+        "correct": 30,
+        "total": 31,
+        "rate": 0.967741935483871
+      },
+      "success": {
+        "correct": 42,
+        "total": 44,
+        "rate": 0.9545454545454546
+      }
+    },
+    "operation:group_count": {
+      "json_valid": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      }
+    },
+    "operation:distinct": {
+      "json_valid": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      }
+    },
+    "operation:year": {
+      "json_valid": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      }
+    },
+    "operation:max": {
+      "json_valid": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      }
+    },
+    "operation:bottom": {
+      "json_valid": {
+        "correct": 5,
+        "total": 5,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 5,
+        "total": 5,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 5,
+        "total": 5,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 5,
+        "total": 5,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 5,
+        "total": 5,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 5,
+        "total": 5,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 5,
+        "total": 5,
+        "rate": 1.0
+      }
+    },
+    "operation:lt": {
+      "json_valid": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      }
+    },
+    "operation:avg": {
+      "json_valid": {
+        "correct": 5,
+        "total": 5,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 5,
+        "total": 5,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 5,
+        "total": 5,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 5,
+        "total": 5,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 5,
+        "total": 5,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 5,
+        "total": 5,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 5,
+        "total": 5,
+        "rate": 1.0
+      }
+    },
+    "operation:sum": {
+      "json_valid": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 3,
+        "total": 4,
+        "rate": 0.75
+      },
+      "sql_equivalent": {
+        "correct": 3,
+        "total": 4,
+        "rate": 0.75
+      },
+      "success": {
+        "correct": 3,
+        "total": 4,
+        "rate": 0.75
+      }
+    },
+    "language:hi": {
+      "json_valid": {
+        "correct": 48,
+        "total": 48,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 48,
+        "total": 48,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 48,
+        "total": 48,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 48,
+        "total": 48,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 47,
+        "total": 48,
+        "rate": 0.9791666666666666
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 29,
+        "rate": 0.9655172413793104
+      },
+      "success": {
+        "correct": 47,
+        "total": 48,
+        "rate": 0.9791666666666666
+      }
+    },
+    "operation:project_eq": {
+      "json_valid": {
+        "correct": 1,
+        "total": 1,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 1,
+        "total": 1,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 1,
+        "total": 1,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 1,
+        "total": 1,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 1,
+        "total": 1,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 1,
+        "total": 1,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 1,
+        "total": 1,
+        "rate": 1.0
+      }
+    },
+    "operation:not_null": {
+      "json_valid": {
+        "correct": 2,
+        "total": 2,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 2,
+        "total": 2,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 2,
+        "total": 2,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 2,
+        "total": 2,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 2,
+        "total": 2,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 2,
+        "total": 2,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 2,
+        "total": 2,
+        "rate": 1.0
+      }
+    },
+    "operation:or": {
+      "json_valid": {
+        "correct": 7,
+        "total": 7,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 7,
+        "total": 7,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 7,
+        "total": 7,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 7,
+        "total": 7,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 7,
+        "total": 7,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 7,
+        "total": 7,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 7,
+        "total": 7,
+        "rate": 1.0
+      }
+    },
+    "operation:date_after": {
+      "json_valid": {
+        "correct": 10,
+        "total": 10,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 10,
+        "total": 10,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 10,
+        "total": 10,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 10,
+        "total": 10,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 10,
+        "total": 10,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 10,
+        "total": 10,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 10,
+        "total": 10,
+        "rate": 1.0
+      }
+    },
+    "operation:top": {
+      "json_valid": {
+        "correct": 9,
+        "total": 9,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 9,
+        "total": 9,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 9,
+        "total": 9,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 9,
+        "total": 9,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 9,
+        "total": 9,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 9,
+        "total": 9,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 9,
+        "total": 9,
+        "rate": 1.0
+      }
+    },
+    "operation:having": {
+      "json_valid": {
+        "correct": 5,
+        "total": 5,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 5,
+        "total": 5,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 5,
+        "total": 5,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 5,
+        "total": 5,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 5,
+        "total": 5,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 5,
+        "total": 5,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 5,
+        "total": 5,
+        "rate": 1.0
+      }
+    },
+    "operation:min": {
+      "json_valid": {
+        "correct": 5,
+        "total": 5,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 5,
+        "total": 5,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 5,
+        "total": 5,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 5,
+        "total": 5,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 5,
+        "total": 5,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 5,
+        "total": 5,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 5,
+        "total": 5,
+        "rate": 1.0
+      }
+    },
+    "operation:gt": {
+      "json_valid": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      }
+    },
+    "backend:mysql": {
+      "json_valid": {
+        "correct": 100,
+        "total": 100,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 100,
+        "total": 100,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 100,
+        "total": 100,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 100,
+        "total": 100,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 98,
+        "total": 100,
+        "rate": 0.98
+      },
+      "sql_equivalent": {
+        "correct": 61,
+        "total": 62,
+        "rate": 0.9838709677419355
+      },
+      "success": {
+        "correct": 98,
+        "total": 100,
+        "rate": 0.98
+      }
+    },
+    "operation:count": {
+      "json_valid": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      }
+    },
+    "operation:group_sum": {
+      "json_valid": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      }
+    },
+    "operation:between": {
+      "json_valid": {
+        "correct": 6,
+        "total": 6,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 6,
+        "total": 6,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 6,
+        "total": 6,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 6,
+        "total": 6,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 6,
+        "total": 6,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 6,
+        "total": 6,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 6,
+        "total": 6,
+        "rate": 1.0
+      }
+    },
+    "operation:sort_asc": {
+      "json_valid": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      }
+    },
+    "operation:sort_desc": {
+      "json_valid": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      }
+    },
+    "operation:sql_error": {
+      "json_valid": {
+        "correct": 9,
+        "total": 9,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 9,
+        "total": 9,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 9,
+        "total": 9,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 9,
+        "total": 9,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 8,
+        "total": 9,
+        "rate": 0.8888888888888888
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 8,
+        "total": 9,
+        "rate": 0.8888888888888888
+      }
+    },
+    "operation:all": {
+      "json_valid": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 3,
+        "total": 4,
+        "rate": 0.75
+      },
+      "sql_equivalent": {
+        "correct": 3,
+        "total": 4,
+        "rate": 0.75
+      },
+      "success": {
+        "correct": 3,
+        "total": 4,
+        "rate": 0.75
+      }
+    },
+    "operation:and": {
+      "json_valid": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      }
+    },
+    "operation:month": {
+      "json_valid": {
+        "correct": 2,
+        "total": 2,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 2,
+        "total": 2,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 2,
+        "total": 2,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 2,
+        "total": 2,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 2,
+        "total": 2,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 2,
+        "total": 2,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 2,
+        "total": 2,
+        "rate": 1.0
+      }
+    },
+    "operation:join_filter": {
+      "json_valid": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      }
+    },
+    "operation:eq": {
+      "json_valid": {
+        "correct": 1,
+        "total": 1,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 1,
+        "total": 1,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 1,
+        "total": 1,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 1,
+        "total": 1,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 1,
+        "total": 1,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 1,
+        "total": 1,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 1,
+        "total": 1,
+        "rate": 1.0
+      }
+    },
+    "operation:list_tables": {
+      "json_valid": {
+        "correct": 12,
+        "total": 12,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 12,
+        "total": 12,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 12,
+        "total": 12,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 12,
+        "total": 12,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 12,
+        "total": 12,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 12,
+        "total": 12,
+        "rate": 1.0
+      }
+    },
+    "operation:gte": {
+      "json_valid": {
+        "correct": 1,
+        "total": 1,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 1,
+        "total": 1,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 1,
+        "total": 1,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 1,
+        "total": 1,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 1,
+        "total": 1,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 1,
+        "total": 1,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 1,
+        "total": 1,
+        "rate": 1.0
+      }
+    },
+    "operation:lte": {
+      "json_valid": {
+        "correct": 1,
+        "total": 1,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 1,
+        "total": 1,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 1,
+        "total": 1,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 1,
+        "total": 1,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 1,
+        "total": 1,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 1,
+        "total": 1,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 1,
+        "total": 1,
+        "rate": 1.0
+      }
+    },
+    "operation:describe": {
+      "json_valid": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 4,
+        "total": 4,
+        "rate": 1.0
+      }
+    },
+    "operation:join": {
+      "json_valid": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 3,
+        "total": 3,
+        "rate": 1.0
+      }
+    },
+    "operation:null": {
+      "json_valid": {
+        "correct": 2,
+        "total": 2,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 2,
+        "total": 2,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 2,
+        "total": 2,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 2,
+        "total": 2,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 2,
+        "total": 2,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 2,
+        "total": 2,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 2,
+        "total": 2,
+        "rate": 1.0
+      }
+    },
+    "operation:contains": {
+      "json_valid": {
+        "correct": 1,
+        "total": 1,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 1,
+        "total": 1,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 1,
+        "total": 1,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 1,
+        "total": 1,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 1,
+        "total": 1,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 1,
+        "total": 1,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 1,
+        "total": 1,
+        "rate": 1.0
+      }
+    },
+    "operation:search": {
+      "json_valid": {
+        "correct": 9,
+        "total": 9,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 9,
+        "total": 9,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 9,
+        "total": 9,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 9,
+        "total": 9,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 9,
+        "total": 9,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 9,
+        "total": 9,
+        "rate": 1.0
+      }
+    },
+    "operation:ticket": {
+      "json_valid": {
+        "correct": 9,
+        "total": 9,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 9,
+        "total": 9,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 9,
+        "total": 9,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 9,
+        "total": 9,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 9,
+        "total": 9,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 9,
+        "total": 9,
+        "rate": 1.0
+      }
+    },
+    "operation:weather": {
+      "json_valid": {
+        "correct": 7,
+        "total": 7,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 7,
+        "total": 7,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 7,
+        "total": 7,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 7,
+        "total": 7,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 7,
+        "total": 7,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 7,
+        "total": 7,
+        "rate": 1.0
+      }
+    },
+    "operation:read_file": {
+      "json_valid": {
+        "correct": 7,
+        "total": 7,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 7,
+        "total": 7,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 7,
+        "total": 7,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 7,
+        "total": 7,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 7,
+        "total": 7,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 7,
+        "total": 7,
+        "rate": 1.0
+      }
+    }
+  }
+}

evaluation/gpu-final-console-metrics.json

@@ -0,0 +1,83 @@
+{
+  "checkpoint_sha256": "9489b5cde69c11f64b3e7031182e8248aa83790f1ad4aeefbf99a4e5e152e692",
+  "frozen_at_utc": "2026-09-10T14:00:36.513800+00:00",
+  "device": "CUDA BF16",
+  "source": "Captured successful remote evaluation console output; raw remote prediction files were not retrieved before SSH began refusing connections. Independent Mac evaluation is packaged separately.",
+  "decoding": "raw greedy; no repair, constrained decoding or teacher fallback",
+  "metrics": {
+    "test": {
+      "json_valid": {
+        "correct": 1196,
+        "total": 1196,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 1191,
+        "total": 1196,
+        "rate": 0.995819397993311
+      },
+      "action_correct": {
+        "correct": 1192,
+        "total": 1196,
+        "rate": 0.9966555183946488
+      },
+      "tool_correct": {
+        "correct": 1088,
+        "total": 1100,
+        "rate": 0.9890909090909091
+      },
+      "arguments_exact": {
+        "correct": 1014,
+        "total": 1100,
+        "rate": 0.9218181818181819
+      },
+      "sql_equivalent": {
+        "correct": 896,
+        "total": 912,
+        "rate": 0.9824561403508771
+      },
+      "success": {
+        "correct": 1106,
+        "total": 1196,
+        "rate": 0.9247491638795987
+      }
+    },
+    "manual": {
+      "json_valid": {
+        "correct": 160,
+        "total": 160,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 159,
+        "total": 160,
+        "rate": 0.99375
+      },
+      "action_correct": {
+        "correct": 159,
+        "total": 160,
+        "rate": 0.99375
+      },
+      "tool_correct": {
+        "correct": 146,
+        "total": 152,
+        "rate": 0.9605263157894737
+      },
+      "arguments_exact": {
+        "correct": 124,
+        "total": 152,
+        "rate": 0.8157894736842105
+      },
+      "sql_equivalent": {
+        "correct": 94,
+        "total": 112,
+        "rate": 0.8392857142857143
+      },
+      "success": {
+        "correct": 132,
+        "total": 160,
+        "rate": 0.825
+      }
+    }
+  }
+}

evaluation/manual-mac.jsonl

@@ -0,0 +1,160 @@
+{"id": "manual_2ad3e251c936135e2005_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "all", "question": "all specimens pls", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"clarify\",\"question\":\"This connection is read-only. Would you like a SELECT query instead?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": false, "tool_correct": false, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_2e2ac3c5dcf95429daaa_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "null", "question": "specimens title null records", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_0b8db53d711baed4f4ed_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "gte", "question": "specimens cost minimum 10 include same", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE cost >= 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY cost ASC LIMIT 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_2ad3e251c936135e2005_en", "language": "en", "backend": "supabase", "operation": "all", "question": "I need everything in specimens.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_efee65268e6dc2bb069e_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "project", "question": "specimens only customer_name show", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT customer_name FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT customer_name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_efee65268e6dc2bb069e_hinglish", "language": "hinglish", "backend": "supabase", "operation": "project", "question": "bas specimens ke customer_name dikha do", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT customer_name FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT customer_name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_0b8db53d711baed4f4ed_hinglish", "language": "hinglish", "backend": "supabase", "operation": "gte", "question": "specimens me cost kam se kam 10 ho", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE cost >= 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE cost >= 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_2e2ac3c5dcf95429daaa_hinglish", "language": "hinglish", "backend": "supabase", "operation": "null", "question": "specimens me jinka title NULL hai wo dikhao", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE price > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_2ad3e251c936135e2005_hinglish", "language": "hinglish", "backend": "supabase", "operation": "all", "question": "specimens ka sara data chahiye", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE display_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_d735c1ac1979ff1e8620_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sum", "question": "inspections price all add", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT SUM(price) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(price) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_0b8db53d711baed4f4ed_en", "language": "en", "backend": "supabase", "operation": "gte", "question": "Include specimens records at 10 or above in cost.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE cost >= 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE cost >= 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_2e2ac3c5dcf95429daaa_en", "language": "en", "backend": "supabase", "operation": "null", "question": "Find specimens records with no title value (NULL).", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE city = 'ApecULL';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_7dd163238936d96d0bd4_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "distinct", "question": "reservations city unique only", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT DISTINCT city FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT city FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_efee65268e6dc2bb069e_en", "language": "en", "backend": "supabase", "operation": "project", "question": "Just the customer_name values from specimens, please.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT customer_name FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT customer_name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_eb982cf071094adaca4f_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "project_eq", "question": "reservations category Delhi only name", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT name FROM reservations WHERE category = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT name FROM reservations WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_d735c1ac1979ff1e8620_en", "language": "en", "backend": "supabase", "operation": "sum", "question": "Add up price across inspections.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT SUM(price) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM inspections WHERE kind = 'Adddd';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_d735c1ac1979ff1e8620_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sum", "question": "inspections ke price ka jod batao", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT SUM(price) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(price) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_c4f25575abfd24e3cd63_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "lte", "question": "reservations cost max 1 equal also", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE cost <= 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY cost DESC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_7dd163238936d96d0bd4_hinglish", "language": "hinglish", "backend": "supabase", "operation": "distinct", "question": "reservations me city ki unique values kya hain", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT DISTINCT city FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT city FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_eb982cf071094adaca4f_hinglish", "language": "hinglish", "backend": "supabase", "operation": "project_eq", "question": "reservations me category Delhi walon ka name batao", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT name FROM reservations WHERE category = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT name FROM reservations WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_eb982cf071094adaca4f_en", "language": "en", "backend": "supabase", "operation": "project_eq", "question": "In reservations, give me name for category Delhi.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT name FROM reservations WHERE category = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT name FROM reservations WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_c4f25575abfd24e3cd63_hinglish", "language": "hinglish", "backend": "supabase", "operation": "lte", "question": "reservations me cost 1 ya usse kam ho", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE cost <= 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE cost <= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_7dd163238936d96d0bd4_en", "language": "en", "backend": "supabase", "operation": "distinct", "question": "Which different city values occur in reservations?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT DISTINCT city FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT city FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_efee65268e6dc2bb069e_hi", "language": "hi", "backend": "supabase", "operation": "project", "question": "specimens से सिर्फ customer_name दिखाना।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT customer_name FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT customer_name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_0b8db53d711baed4f4ed_hi", "language": "hi", "backend": "supabase", "operation": "gte", "question": "specimens में cost कम से कम 10 होना चाहिए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE cost >= 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE cost >= 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_d735c1ac1979ff1e8620_hi", "language": "hi", "backend": "supabase", "operation": "sum", "question": "inspections के price का जोड़ बताओ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT SUM(price) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(price) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_2ad3e251c936135e2005_hi", "language": "hi", "backend": "supabase", "operation": "all", "question": "specimens का सारा डेटा चाहिए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_c4f25575abfd24e3cd63_hi", "language": "hi", "backend": "supabase", "operation": "lte", "question": "reservations में cost 1 या उससे कम हो।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE cost <= 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE cost <= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_c4f25575abfd24e3cd63_en", "language": "en", "backend": "supabase", "operation": "lte", "question": "From reservations, include cost values up to and including 1.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE cost <= 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE cost BETWEEN 1 AND 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_eb982cf071094adaca4f_hi", "language": "hi", "backend": "supabase", "operation": "project_eq", "question": "reservations में category Delhi हो तो उनका name बताओ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT name FROM reservations WHERE category = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT name FROM reservations WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_7dd163238936d96d0bd4_hi", "language": "hi", "backend": "supabase", "operation": "distinct", "question": "reservations में city के अलग-अलग मान कौन से हैं?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT DISTINCT city FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT city FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_2e2ac3c5dcf95429daaa_hi", "language": "hi", "backend": "supabase", "operation": "null", "question": "specimens में जिनका title NULL है वे रिकॉर्ड दिखाओ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE price > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_5a8f2e207d0efe5b018c_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "list_tables", "question": "tables here list", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false}}
+{"id": "manual_5a8f2e207d0efe5b018c_hinglish", "language": "hinglish", "backend": "supabase", "operation": "list_tables", "question": "yahan kaunsi tables hain", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
+{"id": "manual_8ebc9e868ee558a8ddfe_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "ambiguous", "question": "specimens best ones", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
+{"id": "manual_35406fa024e66bcb5dd7_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "lt", "question": "specimens score less 10000", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE score < 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE score < 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_82c104614df1ce6e2762_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "gt", "question": "specimens salary more than 100", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE salary > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE salary > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_82c104614df1ce6e2762_en", "language": "en", "backend": "supabase", "operation": "gt", "question": "Which specimens entries have salary above 100?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE salary > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens WHERE salary > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
+{"id": "manual_35406fa024e66bcb5dd7_en", "language": "en", "backend": "supabase", "operation": "lt", "question": "Find specimens records below 10000 in score.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE score < 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens WHERE score < 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
+{"id": "manual_b564e1a7b1e7fc6d2ab4_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "avg", "question": "specimens avg score tell", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT AVG(score) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT AVG(score) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_5a8f2e207d0efe5b018c_en", "language": "en", "backend": "supabase", "operation": "list_tables", "question": "What tables can I query here?", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}}
+{"id": "manual_8ebc9e868ee558a8ddfe_en", "language": "en", "backend": "supabase", "operation": "ambiguous", "question": "Pick the best entries in specimens.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
+{"id": "manual_8ebc9e868ee558a8ddfe_hinglish", "language": "hinglish", "backend": "supabase", "operation": "ambiguous", "question": "specimens me best records chun lo", "expected": {"action": "clarify", "question": "Best ka matlab kya hai? Kaunsa column aur order?"}, "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
+{"id": "manual_35406fa024e66bcb5dd7_hinglish", "language": "hinglish", "backend": "supabase", "operation": "lt", "question": "specimens me score 10000 se kam wale lao", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE score < 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE score < 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_82c104614df1ce6e2762_hinglish", "language": "hinglish", "backend": "supabase", "operation": "gt", "question": "specimens me salary 100 se upar wale kaun hain", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE salary > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE salary > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_b564e1a7b1e7fc6d2ab4_en", "language": "en", "backend": "supabase", "operation": "avg", "question": "What does score average out to in specimens?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT AVG(score) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT AVG(score) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}}
+{"id": "manual_b564e1a7b1e7fc6d2ab4_hinglish", "language": "hinglish", "backend": "supabase", "operation": "avg", "question": "specimens me score ka average kya hai", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT AVG(score) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT AVG(score) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_40464b00088c9159e3d0_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "count", "question": "inspections total rows how much", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT COUNT(*) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT COUNT(*) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_475287613764159ee5eb_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "group_count", "question": "reservations count per category", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_fd0bab1cc08d7298f131_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "top", "question": "exhibits top 20 by salary big first", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_40464b00088c9159e3d0_hinglish", "language": "hinglish", "backend": "supabase", "operation": "count", "question": "inspections me total kitni rows hain", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT COUNT(*) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT COUNT(*) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_40464b00088c9159e3d0_en", "language": "en", "backend": "supabase", "operation": "count", "question": "How big is inspections, measured in rows?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT COUNT(*) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT COUNT(*) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_fd0bab1cc08d7298f131_hinglish", "language": "hinglish", "backend": "supabase", "operation": "top", "question": "exhibits me sabse bade salary wali 20 rows descending dikhao", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_475287613764159ee5eb_hinglish", "language": "hinglish", "backend": "supabase", "operation": "group_count", "question": "reservations me har category ki count alag batao", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_5a8f2e207d0efe5b018c_hi", "language": "hi", "backend": "supabase", "operation": "list_tables", "question": "यहाँ कौन-कौन सी टेबल हैं?", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens WHERE kind = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false}}
+{"id": "manual_b564e1a7b1e7fc6d2ab4_hi", "language": "hi", "backend": "supabase", "operation": "avg", "question": "specimens में score का औसत क्या है?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT AVG(score) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT AVG(score) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_fd0bab1cc08d7298f131_en", "language": "en", "backend": "supabase", "operation": "top", "question": "Give me 20 entries in exhibits with the largest salary, largest first.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_475287613764159ee5eb_en", "language": "en", "backend": "supabase", "operation": "group_count", "question": "Break down the number of reservations rows by category.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT COUNT(*) FROM reservations WHERE category = 'Areak downs';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_82c104614df1ce6e2762_hi", "language": "hi", "backend": "supabase", "operation": "gt", "question": "specimens में किनका salary 100 से ज़्यादा है?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE salary > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE salary > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_8ebc9e868ee558a8ddfe_hi", "language": "hi", "backend": "supabase", "operation": "ambiguous", "question": "specimens में सबसे अच्छे रिकॉर्ड चुनो।", "expected": {"action": "clarify", "question": "सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?"}, "output": "{\"action\":\"clarify\",\"question\":\"सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
+{"id": "manual_c4210296305105ab7472_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "ambiguous", "question": "specimens best ones", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
+{"id": "manual_9bf8948becf74e81a103_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "list_tables", "question": "tables here list", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false}}
+{"id": "manual_9bf8948becf74e81a103_hinglish", "language": "hinglish", "backend": "mysql", "operation": "list_tables", "question": "yahan kaunsi tables hain", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_8efcf517aee64807a8e5_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "avg", "question": "specimens avg balance tell", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(balance) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_adff4eabe2639c0c128c_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "all", "question": "all inspections pls", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE description IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_35406fa024e66bcb5dd7_hi", "language": "hi", "backend": "supabase", "operation": "lt", "question": "specimens में score 10000 से कम वाले रिकॉर्ड लाओ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE score < 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE score < 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_23c0aa3a4596f995ad2a_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "count", "question": "specimens total rows how much", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_475287613764159ee5eb_hi", "language": "hi", "backend": "supabase", "operation": "group_count", "question": "reservations में हर category की गिनती अलग-अलग बताओ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_23c0aa3a4596f995ad2a_hinglish", "language": "hinglish", "backend": "mysql", "operation": "count", "question": "specimens me total kitni rows hain", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_2164b376f732a1c371ab_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "group_count", "question": "specimens count per kind", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT kind, COUNT(*) FROM specimens GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM specimens GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_40464b00088c9159e3d0_hi", "language": "hi", "backend": "supabase", "operation": "count", "question": "inspections में कुल कितनी पंक्तियाँ हैं?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT COUNT(*) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT COUNT(*) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_8efcf517aee64807a8e5_en", "language": "en", "backend": "mysql", "operation": "avg", "question": "What does balance average out to in specimens?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(balance) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_8efcf517aee64807a8e5_hinglish", "language": "hinglish", "backend": "mysql", "operation": "avg", "question": "specimens me balance ka average kya hai", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(balance) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_c4210296305105ab7472_en", "language": "en", "backend": "mysql", "operation": "ambiguous", "question": "Pick the best entries in specimens.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
+{"id": "manual_c4210296305105ab7472_hinglish", "language": "hinglish", "backend": "mysql", "operation": "ambiguous", "question": "specimens me best records chun lo", "expected": {"action": "clarify", "question": "Best ka matlab kya hai? Kaunsa column aur order?"}, "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
+{"id": "manual_23c0aa3a4596f995ad2a_en", "language": "en", "backend": "mysql", "operation": "count", "question": "How big is specimens, measured in rows?", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_d30264bbc3b08aedbc51_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sum", "question": "exhibits score all add", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT SUM(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT SUM(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_9bf8948becf74e81a103_en", "language": "en", "backend": "mysql", "operation": "list_tables", "question": "What tables can I query here?", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": false, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false, "error": "'sql' is a required property\n\nFailed validating 'required' in schema:\n    {'type': 'object',\n     'properties': {'sql': {'type': 'string'}},\n     'required': ['sql'],\n     'additionalProperties': False}\n\nOn instance:\n    {'schemas': ['public']}"}}
+{"id": "manual_adff4eabe2639c0c128c_en", "language": "en", "backend": "mysql", "operation": "all", "question": "I need everything in inspections.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_b9c176c07809f42564a0_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "top", "question": "inspections top 5 by amount big first", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_c023a5b6f95173ae856d_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "gt", "question": "exhibits score more than 50", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM exhibits WHERE score > 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE score > 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_adff4eabe2639c0c128c_hinglish", "language": "hinglish", "backend": "mysql", "operation": "all", "question": "inspections ka sara data chahiye", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_cab55a7429427fbc5093_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "distinct", "question": "inspections department unique only", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT DISTINCT department FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_e8f90532d34c12209a83_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "gte", "question": "inspections salary minimum 1500 include same", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE salary >= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE salary >= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_c023a5b6f95173ae856d_en", "language": "en", "backend": "mysql", "operation": "gt", "question": "Which exhibits entries have score above 50?", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM exhibits WHERE score > 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE score > 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_affa5e5e14444cd82bb4_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "lt", "question": "reservations salary less 10", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE salary < 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE salary < 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_2164b376f732a1c371ab_hinglish", "language": "hinglish", "backend": "mysql", "operation": "group_count", "question": "specimens me har kind ki count alag batao", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT kind, COUNT(*) FROM specimens GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM specimens GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_d30264bbc3b08aedbc51_en", "language": "en", "backend": "mysql", "operation": "sum", "question": "Add up score across exhibits.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT SUM(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT name FROM exhibits WHERE city = 'Adddd';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false, "error": "no such column: name"}}
+{"id": "manual_d30264bbc3b08aedbc51_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sum", "question": "exhibits ke score ka jod batao", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT SUM(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT SUM(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_b02f421828127e2eb96b_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "lte", "question": "exhibits price max 2 equal also", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits WHERE price <= 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits ORDER BY price DESC LIMIT 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_b9c176c07809f42564a0_hinglish", "language": "hinglish", "backend": "mysql", "operation": "top", "question": "inspections me sabse bade amount wali 5 rows descending dikhao", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_e8f90532d34c12209a83_hinglish", "language": "hinglish", "backend": "mysql", "operation": "gte", "question": "inspections me salary kam se kam 1500 ho", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE salary >= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE salary >= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_2c31add7d7b45843dec1_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "project_eq", "question": "inspections kind Pune only display_name", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT display_name FROM inspections WHERE kind = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM inspections WHERE kind = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_c023a5b6f95173ae856d_hinglish", "language": "hinglish", "backend": "mysql", "operation": "gt", "question": "exhibits me score 50 se upar wale kaun hain", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM exhibits WHERE score > 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE score > 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_2164b376f732a1c371ab_en", "language": "en", "backend": "mysql", "operation": "group_count", "question": "Break down the number of specimens rows by kind.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT kind, COUNT(*) FROM specimens GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM specimens WHERE kind = 'ureak down';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_b02f421828127e2eb96b_hinglish", "language": "hinglish", "backend": "mysql", "operation": "lte", "question": "exhibits me price 2 ya usse kam ho", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits WHERE price <= 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price <= 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_8efcf517aee64807a8e5_hi", "language": "hi", "backend": "mysql", "operation": "avg", "question": "specimens में balance का औसत क्या है?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(balance) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_cab55a7429427fbc5093_hinglish", "language": "hinglish", "backend": "mysql", "operation": "distinct", "question": "inspections me department ki unique values kya hain", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT DISTINCT department FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_b9c176c07809f42564a0_en", "language": "en", "backend": "mysql", "operation": "top", "question": "Give me 5 entries in inspections with the largest amount, largest first.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_a060620c638d2dc3759e_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "project", "question": "reservations only item_name show", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT item_name FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_affa5e5e14444cd82bb4_en", "language": "en", "backend": "mysql", "operation": "lt", "question": "Find reservations records below 10 in salary.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE salary < 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE salary < 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_affa5e5e14444cd82bb4_hinglish", "language": "hinglish", "backend": "mysql", "operation": "lt", "question": "reservations me salary 10 se kam wale lao", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE salary < 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE salary < 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_e8f90532d34c12209a83_en", "language": "en", "backend": "mysql", "operation": "gte", "question": "Include inspections records at 1500 or above in salary.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE salary >= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE salary >= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_04b2e5886e750df09a51_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "null", "question": "reservations display_name null records", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations WHERE display_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE display_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_9bf8948becf74e81a103_hi", "language": "hi", "backend": "mysql", "operation": "list_tables", "question": "यहाँ कौन-कौन सी टेबल हैं?", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE kind = 'Jaipur';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false}}
+{"id": "manual_cab55a7429427fbc5093_en", "language": "en", "backend": "mysql", "operation": "distinct", "question": "Which different department values occur in inspections?", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT DISTINCT department FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_2c31add7d7b45843dec1_en", "language": "en", "backend": "mysql", "operation": "project_eq", "question": "In inspections, give me display_name for kind Pune.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT display_name FROM inspections WHERE kind = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM inspections WHERE kind = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_2c31add7d7b45843dec1_hinglish", "language": "hinglish", "backend": "mysql", "operation": "project_eq", "question": "inspections me kind Pune walon ka display_name batao", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT display_name FROM inspections WHERE kind = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM inspections WHERE kind = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_a060620c638d2dc3759e_hinglish", "language": "hinglish", "backend": "mysql", "operation": "project", "question": "bas reservations ke item_name dikha do", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT item_name FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_b02f421828127e2eb96b_en", "language": "en", "backend": "mysql", "operation": "lte", "question": "From exhibits, include price values up to and including 2.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits WHERE price <= 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price BETWEEN 2 AND 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_b02f421828127e2eb96b_hi", "language": "hi", "backend": "mysql", "operation": "lte", "question": "exhibits में price 2 या उससे कम हो।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits WHERE price <= 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price <= 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_d30264bbc3b08aedbc51_hi", "language": "hi", "backend": "mysql", "operation": "sum", "question": "exhibits के score का जोड़ बताओ।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT SUM(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT SUM(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_04b2e5886e750df09a51_hinglish", "language": "hinglish", "backend": "mysql", "operation": "null", "question": "reservations me jinka display_name NULL hai wo dikhao", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations WHERE display_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE quantity > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_a060620c638d2dc3759e_en", "language": "en", "backend": "mysql", "operation": "project", "question": "Just the item_name values from reservations, please.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT item_name FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_adff4eabe2639c0c128c_hi", "language": "hi", "backend": "mysql", "operation": "all", "question": "inspections का सारा डेटा चाहिए।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_c4210296305105ab7472_hi", "language": "hi", "backend": "mysql", "operation": "ambiguous", "question": "specimens में सबसे अच्छे रिकॉर्ड चुनो।", "expected": {"action": "clarify", "question": "सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?"}, "output": "{\"action\":\"clarify\",\"question\":\"सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}}
+{"id": "manual_23c0aa3a4596f995ad2a_hi", "language": "hi", "backend": "mysql", "operation": "count", "question": "specimens में कुल कितनी पंक्तियाँ हैं?", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_2164b376f732a1c371ab_hi", "language": "hi", "backend": "mysql", "operation": "group_count", "question": "specimens में हर kind की गिनती अलग-अलग बताओ।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT kind, COUNT(*) FROM specimens GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM specimens GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_c023a5b6f95173ae856d_hi", "language": "hi", "backend": "mysql", "operation": "gt", "question": "exhibits में किनका score 50 से ज़्यादा है?", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM exhibits WHERE score > 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE score > 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_e8f90532d34c12209a83_hi", "language": "hi", "backend": "mysql", "operation": "gte", "question": "inspections में salary कम से कम 1500 होना चाहिए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE salary >= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE salary >= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_04b2e5886e750df09a51_en", "language": "en", "backend": "mysql", "operation": "null", "question": "Find reservations records with no display_name value (NULL).", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations WHERE display_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE display_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_2c31add7d7b45843dec1_hi", "language": "hi", "backend": "mysql", "operation": "project_eq", "question": "inspections में kind Pune हो तो उनका display_name बताओ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT display_name FROM inspections WHERE kind = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM inspections WHERE kind = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_fd0bab1cc08d7298f131_hi", "language": "hi", "backend": "supabase", "operation": "top", "question": "exhibits में सबसे ज़्यादा salary वाली 20 पंक्तियाँ घटते क्रम में दिखाओ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_cab55a7429427fbc5093_hi", "language": "hi", "backend": "mysql", "operation": "distinct", "question": "inspections में department के अलग-अलग मान कौन से हैं?", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT DISTINCT department FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_a060620c638d2dc3759e_hi", "language": "hi", "backend": "mysql", "operation": "project", "question": "reservations से सिर्फ item_name दिखाना।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT item_name FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_affa5e5e14444cd82bb4_hi", "language": "hi", "backend": "mysql", "operation": "lt", "question": "reservations में salary 10 से कम वाले रिकॉर्ड लाओ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE salary < 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE salary < 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_04b2e5886e750df09a51_hi", "language": "hi", "backend": "mysql", "operation": "null", "question": "reservations में जिनका display_name NULL है वे रिकॉर्ड दिखाओ।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations WHERE display_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE quantity > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}}
+{"id": "manual_b9c176c07809f42564a0_hi", "language": "hi", "backend": "mysql", "operation": "top", "question": "inspections में सबसे ज़्यादा amount वाली 5 पंक्तियाँ घटते क्रम में दिखाओ।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}}
+{"id": "manual_88b478ce80eed61bdfb4_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "weather", "question": "Jaipur weather celsius pls", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Jaipur", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_88b478ce80eed61bdfb4_hinglish", "language": "hinglish", "backend": "supabase", "operation": "weather", "question": "Jaipur ka weather celsius me batao", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Jaipur", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_88b478ce80eed61bdfb4_en", "language": "en", "backend": "supabase", "operation": "weather", "question": "What is the weather like in Jaipur? Use celsius.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Jaipur", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_88b478ce80eed61bdfb4_hi", "language": "hi", "backend": "supabase", "operation": "weather", "question": "Jaipur का मौसम celsius में बताओ।", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Jaipur", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_4e42bbd40467b2b8c27f_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "search", "question": "docs find SQL joins", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "SQL joins"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"SQL joins\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_8a696e3c3522dd69d5ac_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "ticket", "question": "ticket TKT-4486 details", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-4486"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-4486\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_4e42bbd40467b2b8c27f_en", "language": "en", "backend": "supabase", "operation": "search", "question": "Find documentation about SQL joins.", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "SQL joins"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"SQL joins\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_f73e21e0eb10d6617995_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "read_file", "question": "/config/app.json read pls", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/config/app.json"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_8a696e3c3522dd69d5ac_hinglish", "language": "hinglish", "backend": "supabase", "operation": "ticket", "question": "ticket TKT-4486 ki details lao", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-4486"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-4486\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_4e42bbd40467b2b8c27f_hinglish", "language": "hinglish", "backend": "supabase", "operation": "search", "question": "SQL joins ke bare me docs dhundho", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "SQL joins"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"SQL joins\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_f73e21e0eb10d6617995_hinglish", "language": "hinglish", "backend": "supabase", "operation": "read_file", "question": "/config/app.json file padh kar dikhao", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/config/app.json"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_8a696e3c3522dd69d5ac_en", "language": "en", "backend": "supabase", "operation": "ticket", "question": "Fetch the details for ticket TKT-4486.", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-4486"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-4486\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_f73e21e0eb10d6617995_en", "language": "en", "backend": "supabase", "operation": "read_file", "question": "Open /config/app.json and read its text.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/config/app.json"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_8a696e3c3522dd69d5ac_hi", "language": "hi", "backend": "supabase", "operation": "ticket", "question": "टिकट TKT-4486 का विवरण लाओ।", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-4486"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-4486\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_f73e21e0eb10d6617995_hi", "language": "hi", "backend": "supabase", "operation": "read_file", "question": "/config/app.json फ़ाइल पढ़कर दिखाओ।", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/config/app.json"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_252c6053295d69a93c94_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "search", "question": "docs find database backups", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "database backups"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_5ebc52b3d20bde2076b3_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "ticket", "question": "ticket TKT-6124 details", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-6124"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6124\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_754691bd1eea25d41ccf_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "weather", "question": "Delhi weather fahrenheit pls", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_252c6053295d69a93c94_en", "language": "en", "backend": "mysql", "operation": "search", "question": "Find documentation about database backups.", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "database backups"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_4e42bbd40467b2b8c27f_hi", "language": "hi", "backend": "supabase", "operation": "search", "question": "SQL joins के बारे में दस्तावेज़ खोजो।", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "SQL joins"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"SQL joins\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_5ebc52b3d20bde2076b3_hinglish", "language": "hinglish", "backend": "mysql", "operation": "ticket", "question": "ticket TKT-6124 ki details lao", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-6124"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6124\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_754691bd1eea25d41ccf_hinglish", "language": "hinglish", "backend": "mysql", "operation": "weather", "question": "Delhi ka weather fahrenheit me batao", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_252c6053295d69a93c94_hinglish", "language": "hinglish", "backend": "mysql", "operation": "search", "question": "database backups ke bare me docs dhundho", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "database backups"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_6f3055c580b33340c854_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "read_file", "question": "/notes/setup.txt read pls", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_5ebc52b3d20bde2076b3_en", "language": "en", "backend": "mysql", "operation": "ticket", "question": "Fetch the details for ticket TKT-6124.", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-6124"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6124\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_6f3055c580b33340c854_hinglish", "language": "hinglish", "backend": "mysql", "operation": "read_file", "question": "/notes/setup.txt file padh kar dikhao", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_6f3055c580b33340c854_en", "language": "en", "backend": "mysql", "operation": "read_file", "question": "Open /notes/setup.txt and read its text.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_754691bd1eea25d41ccf_en", "language": "en", "backend": "mysql", "operation": "weather", "question": "What is the weather like in Delhi? Use fahrenheit.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_754691bd1eea25d41ccf_hi", "language": "hi", "backend": "mysql", "operation": "weather", "question": "Delhi का मौसम fahrenheit में बताओ।", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_5ebc52b3d20bde2076b3_hi", "language": "hi", "backend": "mysql", "operation": "ticket", "question": "टिकट TKT-6124 का विवरण लाओ।", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-6124"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6124\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_6f3055c580b33340c854_hi", "language": "hi", "backend": "mysql", "operation": "read_file", "question": "/notes/setup.txt फ़ाइल पढ़कर दिखाओ।", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}
+{"id": "manual_252c6053295d69a93c94_hi", "language": "hi", "backend": "mysql", "operation": "search", "question": "database backups के बारे में दस्तावेज़ खोजो।", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "database backups"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}}

evaluation/manual-mac.summary.json

@@ -0,0 +1,1008 @@
+{
+  "examples": 160,
+  "seconds": 16.877315998077393,
+  "decoding": "raw greedy, no repair, no teacher fallback",
+  "checkpoint_sha256": "9489b5cde69c11f64b3e7031182e8248aa83790f1ad4aeefbf99a4e5e152e692",
+  "sql_metric": "Compile against the supplied schema, then compare results on two generated SQLite fixtures after dialect adaptation; not native MySQL/PostgreSQL execution.",
+  "metrics": {
+    "json_valid": {
+      "correct": 160,
+      "total": 160,
+      "rate": 1.0
+    },
+    "schema_valid": {
+      "correct": 159,
+      "total": 160,
+      "rate": 0.99375
+    },
+    "action_correct": {
+      "correct": 159,
+      "total": 160,
+      "rate": 0.99375
+    },
+    "tool_correct": {
+      "correct": 146,
+      "total": 152,
+      "rate": 0.9605263157894737
+    },
+    "arguments_exact": {
+      "correct": 124,
+      "total": 152,
+      "rate": 0.8157894736842105
+    },
+    "sql_equivalent": {
+      "correct": 94,
+      "total": 112,
+      "rate": 0.8392857142857143
+    },
+    "success": {
+      "correct": 132,
+      "total": 160,
+      "rate": 0.825
+    }
+  },
+  "groups": {
+    "language:noisy_en": {
+      "json_valid": {
+        "correct": 40,
+        "total": 40,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 40,
+        "total": 40,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 39,
+        "total": 40,
+        "rate": 0.975
+      },
+      "tool_correct": {
+        "correct": 35,
+        "total": 38,
+        "rate": 0.9210526315789473
+      },
+      "arguments_exact": {
+        "correct": 31,
+        "total": 38,
+        "rate": 0.8157894736842105
+      },
+      "sql_equivalent": {
+        "correct": 23,
+        "total": 28,
+        "rate": 0.8214285714285714
+      },
+      "success": {
+        "correct": 33,
+        "total": 40,
+        "rate": 0.825
+      }
+    },
+    "backend:supabase": {
+      "json_valid": {
+        "correct": 80,
+        "total": 80,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 80,
+        "total": 80,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 79,
+        "total": 80,
+        "rate": 0.9875
+      },
+      "tool_correct": {
+        "correct": 73,
+        "total": 76,
+        "rate": 0.9605263157894737
+      },
+      "arguments_exact": {
+        "correct": 59,
+        "total": 76,
+        "rate": 0.7763157894736842
+      },
+      "sql_equivalent": {
+        "correct": 46,
+        "total": 56,
+        "rate": 0.8214285714285714
+      },
+      "success": {
+        "correct": 63,
+        "total": 80,
+        "rate": 0.7875
+      }
+    },
+    "operation:all": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 7,
+        "total": 8,
+        "rate": 0.875
+      },
+      "tool_correct": {
+        "correct": 7,
+        "total": 8,
+        "rate": 0.875
+      },
+      "arguments_exact": {
+        "correct": 5,
+        "total": 8,
+        "rate": 0.625
+      },
+      "sql_equivalent": {
+        "correct": 5,
+        "total": 8,
+        "rate": 0.625
+      },
+      "success": {
+        "correct": 5,
+        "total": 8,
+        "rate": 0.625
+      }
+    },
+    "operation:null": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 2,
+        "total": 8,
+        "rate": 0.25
+      },
+      "sql_equivalent": {
+        "correct": 2,
+        "total": 8,
+        "rate": 0.25
+      },
+      "success": {
+        "correct": 2,
+        "total": 8,
+        "rate": 0.25
+      }
+    },
+    "operation:gte": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 7,
+        "total": 8,
+        "rate": 0.875
+      },
+      "sql_equivalent": {
+        "correct": 7,
+        "total": 8,
+        "rate": 0.875
+      },
+      "success": {
+        "correct": 7,
+        "total": 8,
+        "rate": 0.875
+      }
+    },
+    "language:en": {
+      "json_valid": {
+        "correct": 40,
+        "total": 40,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 39,
+        "total": 40,
+        "rate": 0.975
+      },
+      "action_correct": {
+        "correct": 40,
+        "total": 40,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 37,
+        "total": 38,
+        "rate": 0.9736842105263158
+      },
+      "arguments_exact": {
+        "correct": 25,
+        "total": 38,
+        "rate": 0.6578947368421053
+      },
+      "sql_equivalent": {
+        "correct": 20,
+        "total": 28,
+        "rate": 0.7142857142857143
+      },
+      "success": {
+        "correct": 27,
+        "total": 40,
+        "rate": 0.675
+      }
+    },
+    "operation:project": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      }
+    },
+    "language:hinglish": {
+      "json_valid": {
+        "correct": 40,
+        "total": 40,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 40,
+        "total": 40,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 40,
+        "total": 40,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 38,
+        "total": 38,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 34,
+        "total": 38,
+        "rate": 0.8947368421052632
+      },
+      "sql_equivalent": {
+        "correct": 25,
+        "total": 28,
+        "rate": 0.8928571428571429
+      },
+      "success": {
+        "correct": 36,
+        "total": 40,
+        "rate": 0.9
+      }
+    },
+    "operation:sum": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 6,
+        "total": 8,
+        "rate": 0.75
+      },
+      "sql_equivalent": {
+        "correct": 6,
+        "total": 8,
+        "rate": 0.75
+      },
+      "success": {
+        "correct": 6,
+        "total": 8,
+        "rate": 0.75
+      }
+    },
+    "operation:distinct": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      }
+    },
+    "operation:project_eq": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      }
+    },
+    "operation:lte": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 4,
+        "total": 8,
+        "rate": 0.5
+      },
+      "sql_equivalent": {
+        "correct": 4,
+        "total": 8,
+        "rate": 0.5
+      },
+      "success": {
+        "correct": 4,
+        "total": 8,
+        "rate": 0.5
+      }
+    },
+    "language:hi": {
+      "json_valid": {
+        "correct": 40,
+        "total": 40,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 40,
+        "total": 40,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 40,
+        "total": 40,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 36,
+        "total": 38,
+        "rate": 0.9473684210526315
+      },
+      "arguments_exact": {
+        "correct": 34,
+        "total": 38,
+        "rate": 0.8947368421052632
+      },
+      "sql_equivalent": {
+        "correct": 26,
+        "total": 28,
+        "rate": 0.9285714285714286
+      },
+      "success": {
+        "correct": 36,
+        "total": 40,
+        "rate": 0.9
+      }
+    },
+    "operation:list_tables": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 7,
+        "total": 8,
+        "rate": 0.875
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 3,
+        "total": 8,
+        "rate": 0.375
+      },
+      "arguments_exact": {
+        "correct": 1,
+        "total": 8,
+        "rate": 0.125
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 1,
+        "total": 8,
+        "rate": 0.125
+      }
+    },
+    "operation:ambiguous": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "arguments_exact": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      }
+    },
+    "operation:lt": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 7,
+        "total": 8,
+        "rate": 0.875
+      },
+      "sql_equivalent": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 7,
+        "total": 8,
+        "rate": 0.875
+      }
+    },
+    "operation:gt": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 7,
+        "total": 8,
+        "rate": 0.875
+      },
+      "sql_equivalent": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 7,
+        "total": 8,
+        "rate": 0.875
+      }
+    },
+    "operation:avg": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 7,
+        "total": 8,
+        "rate": 0.875
+      },
+      "sql_equivalent": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 7,
+        "total": 8,
+        "rate": 0.875
+      }
+    },
+    "operation:count": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      }
+    },
+    "operation:group_count": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 6,
+        "total": 8,
+        "rate": 0.75
+      },
+      "sql_equivalent": {
+        "correct": 6,
+        "total": 8,
+        "rate": 0.75
+      },
+      "success": {
+        "correct": 6,
+        "total": 8,
+        "rate": 0.75
+      }
+    },
+    "operation:top": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      }
+    },
+    "backend:mysql": {
+      "json_valid": {
+        "correct": 80,
+        "total": 80,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 79,
+        "total": 80,
+        "rate": 0.9875
+      },
+      "action_correct": {
+        "correct": 80,
+        "total": 80,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 73,
+        "total": 76,
+        "rate": 0.9605263157894737
+      },
+      "arguments_exact": {
+        "correct": 65,
+        "total": 76,
+        "rate": 0.8552631578947368
+      },
+      "sql_equivalent": {
+        "correct": 48,
+        "total": 56,
+        "rate": 0.8571428571428571
+      },
+      "success": {
+        "correct": 69,
+        "total": 80,
+        "rate": 0.8625
+      }
+    },
+    "operation:weather": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      }
+    },
+    "operation:search": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      }
+    },
+    "operation:ticket": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      }
+    },
+    "operation:read_file": {
+      "json_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 8,
+        "total": 8,
+        "rate": 1.0
+      }
+    }
+  }
+}

evaluation/test-mac.summary.json

@@ -0,0 +1,1933 @@
+{
+  "examples": 1196,
+  "seconds": 141.622967004776,
+  "decoding": "raw greedy, no repair, no teacher fallback",
+  "checkpoint_sha256": "9489b5cde69c11f64b3e7031182e8248aa83790f1ad4aeefbf99a4e5e152e692",
+  "sql_metric": "Compile against the supplied schema, then compare results on two generated SQLite fixtures after dialect adaptation; not native MySQL/PostgreSQL execution.",
+  "metrics": {
+    "json_valid": {
+      "correct": 1196,
+      "total": 1196,
+      "rate": 1.0
+    },
+    "schema_valid": {
+      "correct": 1191,
+      "total": 1196,
+      "rate": 0.995819397993311
+    },
+    "action_correct": {
+      "correct": 1192,
+      "total": 1196,
+      "rate": 0.9966555183946488
+    },
+    "tool_correct": {
+      "correct": 1088,
+      "total": 1100,
+      "rate": 0.9890909090909091
+    },
+    "arguments_exact": {
+      "correct": 1014,
+      "total": 1100,
+      "rate": 0.9218181818181819
+    },
+    "sql_equivalent": {
+      "correct": 896,
+      "total": 912,
+      "rate": 0.9824561403508771
+    },
+    "success": {
+      "correct": 1106,
+      "total": 1196,
+      "rate": 0.9247491638795987
+    }
+  },
+  "groups": {
+    "language:noisy_en": {
+      "json_valid": {
+        "correct": 299,
+        "total": 299,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 298,
+        "total": 299,
+        "rate": 0.9966555183946488
+      },
+      "action_correct": {
+        "correct": 299,
+        "total": 299,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 274,
+        "total": 275,
+        "rate": 0.9963636363636363
+      },
+      "arguments_exact": {
+        "correct": 252,
+        "total": 275,
+        "rate": 0.9163636363636364
+      },
+      "sql_equivalent": {
+        "correct": 228,
+        "total": 228,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 276,
+        "total": 299,
+        "rate": 0.9230769230769231
+      }
+    },
+    "backend:supabase": {
+      "json_valid": {
+        "correct": 588,
+        "total": 588,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 588,
+        "total": 588,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 588,
+        "total": 588,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 541,
+        "total": 544,
+        "rate": 0.9944852941176471
+      },
+      "arguments_exact": {
+        "correct": 480,
+        "total": 544,
+        "rate": 0.8823529411764706
+      },
+      "sql_equivalent": {
+        "correct": 429,
+        "total": 436,
+        "rate": 0.9839449541284404
+      },
+      "success": {
+        "correct": 524,
+        "total": 588,
+        "rate": 0.891156462585034
+      }
+    },
+    "operation:missing_schema": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 20,
+        "total": 24,
+        "rate": 0.8333333333333334
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 20,
+        "total": 24,
+        "rate": 0.8333333333333334
+      }
+    },
+    "language:en": {
+      "json_valid": {
+        "correct": 299,
+        "total": 299,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 298,
+        "total": 299,
+        "rate": 0.9966555183946488
+      },
+      "action_correct": {
+        "correct": 297,
+        "total": 299,
+        "rate": 0.9933110367892977
+      },
+      "tool_correct": {
+        "correct": 274,
+        "total": 275,
+        "rate": 0.9963636363636363
+      },
+      "arguments_exact": {
+        "correct": 246,
+        "total": 275,
+        "rate": 0.8945454545454545
+      },
+      "sql_equivalent": {
+        "correct": 222,
+        "total": 228,
+        "rate": 0.9736842105263158
+      },
+      "success": {
+        "correct": 268,
+        "total": 299,
+        "rate": 0.8963210702341137
+      }
+    },
+    "language:hinglish": {
+      "json_valid": {
+        "correct": 299,
+        "total": 299,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 297,
+        "total": 299,
+        "rate": 0.9933110367892977
+      },
+      "action_correct": {
+        "correct": 297,
+        "total": 299,
+        "rate": 0.9933110367892977
+      },
+      "tool_correct": {
+        "correct": 273,
+        "total": 275,
+        "rate": 0.9927272727272727
+      },
+      "arguments_exact": {
+        "correct": 262,
+        "total": 275,
+        "rate": 0.9527272727272728
+      },
+      "sql_equivalent": {
+        "correct": 228,
+        "total": 228,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 284,
+        "total": 299,
+        "rate": 0.9498327759197325
+      }
+    },
+    "operation:sql_error": {
+      "json_valid": {
+        "correct": 20,
+        "total": 20,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 20,
+        "total": 20,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 20,
+        "total": 20,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 20,
+        "total": 20,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 12,
+        "total": 20,
+        "rate": 0.6
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 12,
+        "total": 20,
+        "rate": 0.6
+      }
+    },
+    "backend:mysql": {
+      "json_valid": {
+        "correct": 608,
+        "total": 608,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 603,
+        "total": 608,
+        "rate": 0.9917763157894737
+      },
+      "action_correct": {
+        "correct": 604,
+        "total": 608,
+        "rate": 0.993421052631579
+      },
+      "tool_correct": {
+        "correct": 547,
+        "total": 556,
+        "rate": 0.9838129496402878
+      },
+      "arguments_exact": {
+        "correct": 534,
+        "total": 556,
+        "rate": 0.960431654676259
+      },
+      "sql_equivalent": {
+        "correct": 467,
+        "total": 476,
+        "rate": 0.9810924369747899
+      },
+      "success": {
+        "correct": 582,
+        "total": 608,
+        "rate": 0.9572368421052632
+      }
+    },
+    "operation:list_tables": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 12,
+        "total": 24,
+        "rate": 0.5
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 12,
+        "total": 24,
+        "rate": 0.5
+      }
+    },
+    "operation:all": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:count": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 23,
+        "total": 28,
+        "rate": 0.8214285714285714
+      },
+      "sql_equivalent": {
+        "correct": 23,
+        "total": 28,
+        "rate": 0.8214285714285714
+      },
+      "success": {
+        "correct": 23,
+        "total": 28,
+        "rate": 0.8214285714285714
+      }
+    },
+    "operation:avg": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 26,
+        "total": 28,
+        "rate": 0.9285714285714286
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 26,
+        "total": 28,
+        "rate": 0.9285714285714286
+      }
+    },
+    "operation:describe": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 23,
+        "total": 24,
+        "rate": 0.9583333333333334
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 23,
+        "total": 24,
+        "rate": 0.9583333333333334
+      },
+      "arguments_exact": {
+        "correct": 19,
+        "total": 24,
+        "rate": 0.7916666666666666
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 19,
+        "total": 24,
+        "rate": 0.7916666666666666
+      }
+    },
+    "operation:project": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:gte": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 26,
+        "total": 28,
+        "rate": 0.9285714285714286
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 26,
+        "total": 28,
+        "rate": 0.9285714285714286
+      }
+    },
+    "operation:min": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 24,
+        "total": 28,
+        "rate": 0.8571428571428571
+      },
+      "sql_equivalent": {
+        "correct": 24,
+        "total": 28,
+        "rate": 0.8571428571428571
+      },
+      "success": {
+        "correct": 24,
+        "total": 28,
+        "rate": 0.8571428571428571
+      }
+    },
+    "operation:null": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 26,
+        "total": 28,
+        "rate": 0.9285714285714286
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 26,
+        "total": 28,
+        "rate": 0.9285714285714286
+      }
+    },
+    "operation:write": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 20,
+        "total": 24,
+        "rate": 0.8333333333333334
+      },
+      "tool_correct": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "arguments_exact": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 20,
+        "total": 24,
+        "rate": 0.8333333333333334
+      }
+    },
+    "operation:or": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:count_eq": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 25,
+        "total": 28,
+        "rate": 0.8928571428571429
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 25,
+        "total": 28,
+        "rate": 0.8928571428571429
+      }
+    },
+    "operation:contains": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      }
+    },
+    "operation:sum": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 26,
+        "total": 28,
+        "rate": 0.9285714285714286
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 26,
+        "total": 28,
+        "rate": 0.9285714285714286
+      }
+    },
+    "operation:sort_desc": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 26,
+        "total": 28,
+        "rate": 0.9285714285714286
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 26,
+        "total": 28,
+        "rate": 0.9285714285714286
+      }
+    },
+    "operation:eq": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:project_eq": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "language:hi": {
+      "json_valid": {
+        "correct": 299,
+        "total": 299,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 298,
+        "total": 299,
+        "rate": 0.9966555183946488
+      },
+      "action_correct": {
+        "correct": 299,
+        "total": 299,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 267,
+        "total": 275,
+        "rate": 0.9709090909090909
+      },
+      "arguments_exact": {
+        "correct": 254,
+        "total": 275,
+        "rate": 0.9236363636363636
+      },
+      "sql_equivalent": {
+        "correct": 218,
+        "total": 228,
+        "rate": 0.956140350877193
+      },
+      "success": {
+        "correct": 278,
+        "total": 299,
+        "rate": 0.9297658862876255
+      }
+    },
+    "operation:year": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 26,
+        "total": 28,
+        "rate": 0.9285714285714286
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 26,
+        "total": 28,
+        "rate": 0.9285714285714286
+      }
+    },
+    "operation:bottom": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:distinct": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:ambiguous": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "arguments_exact": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      }
+    },
+    "operation:max": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 26,
+        "total": 28,
+        "rate": 0.9285714285714286
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 26,
+        "total": 28,
+        "rate": 0.9285714285714286
+      }
+    },
+    "operation:group_count": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:sort_asc": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:not_null": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 26,
+        "total": 28,
+        "rate": 0.9285714285714286
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 26,
+        "total": 28,
+        "rate": 0.9285714285714286
+      }
+    },
+    "operation:lte": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:lt": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 26,
+        "total": 28,
+        "rate": 0.9285714285714286
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 26,
+        "total": 28,
+        "rate": 0.9285714285714286
+      }
+    },
+    "operation:group_sum": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:between": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 21,
+        "total": 28,
+        "rate": 0.75
+      },
+      "arguments_exact": {
+        "correct": 21,
+        "total": 28,
+        "rate": 0.75
+      },
+      "sql_equivalent": {
+        "correct": 21,
+        "total": 28,
+        "rate": 0.75
+      },
+      "success": {
+        "correct": 21,
+        "total": 28,
+        "rate": 0.75
+      }
+    },
+    "operation:month": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 24,
+        "total": 28,
+        "rate": 0.8571428571428571
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 24,
+        "total": 28,
+        "rate": 0.8571428571428571
+      }
+    },
+    "operation:and": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 26,
+        "total": 28,
+        "rate": 0.9285714285714286
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 26,
+        "total": 28,
+        "rate": 0.9285714285714286
+      }
+    },
+    "operation:having": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 20,
+        "total": 28,
+        "rate": 0.7142857142857143
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 20,
+        "total": 28,
+        "rate": 0.7142857142857143
+      }
+    },
+    "operation:missing_value": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "arguments_exact": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      }
+    },
+    "operation:final": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "arguments_exact": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      }
+    },
+    "operation:date_after": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:gt": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 26,
+        "total": 28,
+        "rate": 0.9285714285714286
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 26,
+        "total": 28,
+        "rate": 0.9285714285714286
+      }
+    },
+    "operation:top": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:join_filter": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      }
+    },
+    "operation:join": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      }
+    },
+    "operation:weather": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      }
+    },
+    "operation:search": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 20,
+        "total": 24,
+        "rate": 0.8333333333333334
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 20,
+        "total": 24,
+        "rate": 0.8333333333333334
+      },
+      "arguments_exact": {
+        "correct": 20,
+        "total": 24,
+        "rate": 0.8333333333333334
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 20,
+        "total": 24,
+        "rate": 0.8333333333333334
+      }
+    },
+    "operation:ticket": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      }
+    },
+    "operation:read_file": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      }
+    }
+  }
+}

evaluation/validation.summary.json

@@ -0,0 +1,1933 @@
+{
+  "examples": 1200,
+  "seconds": 33.96494126319885,
+  "decoding": "raw greedy, no repair, no teacher fallback",
+  "checkpoint_sha256": "9489b5cde69c11f64b3e7031182e8248aa83790f1ad4aeefbf99a4e5e152e692",
+  "sql_metric": "Dialect parsing and equivalent results on two generated SQLite fixtures after dialect adaptation; not native MySQL/PostgreSQL execution.",
+  "metrics": {
+    "json_valid": {
+      "correct": 1200,
+      "total": 1200,
+      "rate": 1.0
+    },
+    "schema_valid": {
+      "correct": 1199,
+      "total": 1200,
+      "rate": 0.9991666666666666
+    },
+    "action_correct": {
+      "correct": 1193,
+      "total": 1200,
+      "rate": 0.9941666666666666
+    },
+    "tool_correct": {
+      "correct": 1103,
+      "total": 1104,
+      "rate": 0.9990942028985508
+    },
+    "arguments_exact": {
+      "correct": 1085,
+      "total": 1104,
+      "rate": 0.9827898550724637
+    },
+    "sql_equivalent": {
+      "correct": 899,
+      "total": 912,
+      "rate": 0.9857456140350878
+    },
+    "success": {
+      "correct": 1174,
+      "total": 1200,
+      "rate": 0.9783333333333334
+    }
+  },
+  "groups": {
+    "language:noisy_en": {
+      "json_valid": {
+        "correct": 300,
+        "total": 300,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 300,
+        "total": 300,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 300,
+        "total": 300,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 276,
+        "total": 276,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 275,
+        "total": 276,
+        "rate": 0.9963768115942029
+      },
+      "sql_equivalent": {
+        "correct": 227,
+        "total": 228,
+        "rate": 0.9956140350877193
+      },
+      "success": {
+        "correct": 299,
+        "total": 300,
+        "rate": 0.9966666666666667
+      }
+    },
+    "backend:supabase": {
+      "json_valid": {
+        "correct": 604,
+        "total": 604,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 603,
+        "total": 604,
+        "rate": 0.9983443708609272
+      },
+      "action_correct": {
+        "correct": 601,
+        "total": 604,
+        "rate": 0.9950331125827815
+      },
+      "tool_correct": {
+        "correct": 559,
+        "total": 560,
+        "rate": 0.9982142857142857
+      },
+      "arguments_exact": {
+        "correct": 551,
+        "total": 560,
+        "rate": 0.9839285714285714
+      },
+      "sql_equivalent": {
+        "correct": 461,
+        "total": 468,
+        "rate": 0.9850427350427351
+      },
+      "success": {
+        "correct": 592,
+        "total": 604,
+        "rate": 0.9801324503311258
+      }
+    },
+    "operation:missing_schema": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      }
+    },
+    "language:en": {
+      "json_valid": {
+        "correct": 300,
+        "total": 300,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 300,
+        "total": 300,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 294,
+        "total": 300,
+        "rate": 0.98
+      },
+      "tool_correct": {
+        "correct": 276,
+        "total": 276,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 268,
+        "total": 276,
+        "rate": 0.9710144927536232
+      },
+      "sql_equivalent": {
+        "correct": 222,
+        "total": 228,
+        "rate": 0.9736842105263158
+      },
+      "success": {
+        "correct": 286,
+        "total": 300,
+        "rate": 0.9533333333333334
+      }
+    },
+    "language:hinglish": {
+      "json_valid": {
+        "correct": 300,
+        "total": 300,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 299,
+        "total": 300,
+        "rate": 0.9966666666666667
+      },
+      "action_correct": {
+        "correct": 300,
+        "total": 300,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 275,
+        "total": 276,
+        "rate": 0.9963768115942029
+      },
+      "arguments_exact": {
+        "correct": 268,
+        "total": 276,
+        "rate": 0.9710144927536232
+      },
+      "sql_equivalent": {
+        "correct": 223,
+        "total": 228,
+        "rate": 0.9780701754385965
+      },
+      "success": {
+        "correct": 292,
+        "total": 300,
+        "rate": 0.9733333333333334
+      }
+    },
+    "backend:mysql": {
+      "json_valid": {
+        "correct": 596,
+        "total": 596,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 596,
+        "total": 596,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 592,
+        "total": 596,
+        "rate": 0.9932885906040269
+      },
+      "tool_correct": {
+        "correct": 544,
+        "total": 544,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 534,
+        "total": 544,
+        "rate": 0.9816176470588235
+      },
+      "sql_equivalent": {
+        "correct": 438,
+        "total": 444,
+        "rate": 0.9864864864864865
+      },
+      "success": {
+        "correct": 582,
+        "total": 596,
+        "rate": 0.9765100671140939
+      }
+    },
+    "operation:all": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:write": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 18,
+        "total": 24,
+        "rate": 0.75
+      },
+      "tool_correct": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "arguments_exact": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 18,
+        "total": 24,
+        "rate": 0.75
+      }
+    },
+    "operation:distinct": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:max": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:min": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:avg": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:group_count": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 25,
+        "total": 28,
+        "rate": 0.8928571428571429
+      },
+      "sql_equivalent": {
+        "correct": 25,
+        "total": 28,
+        "rate": 0.8928571428571429
+      },
+      "success": {
+        "correct": 25,
+        "total": 28,
+        "rate": 0.8928571428571429
+      }
+    },
+    "language:hi": {
+      "json_valid": {
+        "correct": 300,
+        "total": 300,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 300,
+        "total": 300,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 299,
+        "total": 300,
+        "rate": 0.9966666666666667
+      },
+      "tool_correct": {
+        "correct": 276,
+        "total": 276,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 274,
+        "total": 276,
+        "rate": 0.9927536231884058
+      },
+      "sql_equivalent": {
+        "correct": 227,
+        "total": 228,
+        "rate": 0.9956140350877193
+      },
+      "success": {
+        "correct": 297,
+        "total": 300,
+        "rate": 0.99
+      }
+    },
+    "operation:sum": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 27,
+        "total": 28,
+        "rate": 0.9642857142857143
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 27,
+        "total": 28,
+        "rate": 0.9642857142857143
+      }
+    },
+    "operation:count_eq": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:count": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 22,
+        "total": 28,
+        "rate": 0.7857142857142857
+      },
+      "sql_equivalent": {
+        "correct": 22,
+        "total": 28,
+        "rate": 0.7857142857142857
+      },
+      "success": {
+        "correct": 22,
+        "total": 28,
+        "rate": 0.7857142857142857
+      }
+    },
+    "operation:sql_error": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 20,
+        "total": 24,
+        "rate": 0.8333333333333334
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 20,
+        "total": 24,
+        "rate": 0.8333333333333334
+      }
+    },
+    "operation:project": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 24,
+        "total": 28,
+        "rate": 0.8571428571428571
+      },
+      "sql_equivalent": {
+        "correct": 24,
+        "total": 28,
+        "rate": 0.8571428571428571
+      },
+      "success": {
+        "correct": 24,
+        "total": 28,
+        "rate": 0.8571428571428571
+      }
+    },
+    "operation:lt": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:top": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:group_sum": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:sort_desc": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:ambiguous": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "arguments_exact": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      }
+    },
+    "operation:gte": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:eq": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:or": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:month": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:project_eq": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:contains": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      }
+    },
+    "operation:not_null": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:gt": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:lte": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:bottom": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:having": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:sort_asc": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:null": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:year": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:missing_value": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "arguments_exact": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      }
+    },
+    "operation:and": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:date_after": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:final": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 23,
+        "total": 24,
+        "rate": 0.9583333333333334
+      },
+      "tool_correct": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "arguments_exact": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 23,
+        "total": 24,
+        "rate": 0.9583333333333334
+      }
+    },
+    "operation:list_tables": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      }
+    },
+    "operation:describe": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      }
+    },
+    "operation:join": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      }
+    },
+    "operation:join_filter": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      }
+    },
+    "operation:between": {
+      "json_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      },
+      "success": {
+        "correct": 28,
+        "total": 28,
+        "rate": 1.0
+      }
+    },
+    "operation:search": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      }
+    },
+    "operation:weather": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      }
+    },
+    "operation:ticket": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "arguments_exact": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      }
+    },
+    "operation:read_file": {
+      "json_valid": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "schema_valid": {
+        "correct": 23,
+        "total": 24,
+        "rate": 0.9583333333333334
+      },
+      "action_correct": {
+        "correct": 24,
+        "total": 24,
+        "rate": 1.0
+      },
+      "tool_correct": {
+        "correct": 23,
+        "total": 24,
+        "rate": 0.9583333333333334
+      },
+      "arguments_exact": {
+        "correct": 23,
+        "total": 24,
+        "rate": 0.9583333333333334
+      },
+      "sql_equivalent": {
+        "correct": 0,
+        "total": 0,
+        "rate": 0.0
+      },
+      "success": {
+        "correct": 23,
+        "total": 24,
+        "rate": 0.9583333333333334
+      }
+    }
+  }
+}

examples/context-mysql.json

@@ -0,0 +1,59 @@
+{
+  "backend": "mysql",
+  "project_id": "demo",
+  "schema": [
+    "CREATE TABLE customers (id INTEGER PRIMARY KEY, name TEXT, city TEXT, amount REAL);"
+  ],
+  "tools": [
+    {
+      "name": "query",
+      "description": "Execute a read-only MySQL SELECT query.",
+      "inputSchema": {
+        "type": "object",
+        "properties": {
+          "query": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "query"
+        ],
+        "additionalProperties": false
+      }
+    },
+    {
+      "name": "list_tables",
+      "description": "List database tables and their columns.",
+      "inputSchema": {
+        "type": "object",
+        "properties": {
+          "schemas": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          }
+        },
+        "required": [],
+        "additionalProperties": false
+      }
+    },
+    {
+      "name": "describe_table",
+      "description": "Inspect columns and types for one table.",
+      "inputSchema": {
+        "type": "object",
+        "properties": {
+          "table": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "table"
+        ],
+        "additionalProperties": false
+      }
+    }
+  ],
+  "policy": "Read-only database access. Use provided tools. Ask when required information is missing."
+}

examples/context-supabase.json

@@ -0,0 +1,43 @@
+{
+  "backend": "supabase",
+  "project_id": "demo",
+  "schema": [
+    "CREATE TABLE customers (id INTEGER PRIMARY KEY, name TEXT, city TEXT, amount REAL);"
+  ],
+  "tools": [
+    {
+      "name": "execute_sql",
+      "description": "Execute a read-only PostgreSQL SELECT query.",
+      "inputSchema": {
+        "type": "object",
+        "properties": {
+          "query": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "query"
+        ],
+        "additionalProperties": false
+      }
+    },
+    {
+      "name": "list_tables",
+      "description": "List database tables and their columns.",
+      "inputSchema": {
+        "type": "object",
+        "properties": {
+          "schemas": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          }
+        },
+        "required": [],
+        "additionalProperties": false
+      }
+    }
+  ],
+  "policy": "Read-only database access. Use provided tools. Ask when required information is missing."
+}

grounding-stats.json

@@ -0,0 +1,5 @@
+{
+  "layout": 8826,
+  "runtime_names": 13635,
+  "city_language": 4800
+}

manifest.json

@@ -0,0 +1,279 @@
+{
+  "parameters": 139738113,
+  "format": "Custom native PyTorch; see tinyquery/model.py, not a Transformers AutoModel checkpoint",
+  "random_initialization": true,
+  "files": {
+    "LICENSE": {
+      "bytes": 11544,
+      "sha256": "bbedc3fda3305820b977265f01b8619d87570a6739de3a5582c3464840f1e57a"
+    },
+    "README.md": {
+      "bytes": 9919,
+      "sha256": "34cd941a5170b840cbfa0450a741561a42b0ee8fae4b9fa26031ffe1026ae5dd"
+    },
+    "checkpoint-info.json": {
+      "bytes": 144,
+      "sha256": "9626ef405296d0772a9df05af70e96dddebcd1d0840d29180855b65887be589a"
+    },
+    "config.json": {
+      "bytes": 170,
+      "sha256": "94054411ca3c62c3b500405c4ab8c45ecd6d713d88dc95f726908ca8b9411df1"
+    },
+    "docs/architecture.md": {
+      "bytes": 4774,
+      "sha256": "497a2b76330e203dcbcd4e38fd9b03a1adcad2197e1e6bb545210b34fc3574ad"
+    },
+    "docs/experiment.md": {
+      "bytes": 5988,
+      "sha256": "1bba1663c1c927ca91299dedcfc200055420677df357829fd5271bd1ba8fe953"
+    },
+    "evaluation/baseline-manual.jsonl": {
+      "bytes": 64508,
+      "sha256": "7c45ebd7aa01711b10f0d15e47e1c46240c29acae0d70f8392693e7e33207020"
+    },
+    "evaluation/baseline-manual.summary.json": {
+      "bytes": 19377,
+      "sha256": "6f7ac7ea1ab6e2b1b3297bb3393f6e4fd21caefa7962a2d3983662807b6fb527"
+    },
+    "evaluation/baseline-test.jsonl": {
+      "bytes": 486708,
+      "sha256": "630ec167061e963e4557706fb74282dbd3d5ceb2e0951dd6cfc3f8c0b80cff47"
+    },
+    "evaluation/baseline-test.summary.json": {
+      "bytes": 38746,
+      "sha256": "80bb5910474dc7103c73265d8268f4de9b40209a964b54c5a33d9345d5f0d638"
+    },
+    "evaluation/development.summary.json": {
+      "bytes": 34207,
+      "sha256": "d9b17d2ab010df858674362a4cdf5275939d8728b594769637fa5b7954a1f2cd"
+    },
+    "evaluation/gpu-final-console-metrics.json": {
+      "bytes": 2062,
+      "sha256": "1760d2862fe481bb65d8bfad8110541d315e740ec1546996e777b547f4291b6b"
+    },
+    "evaluation/manual-mac.jsonl": {
+      "bytes": 98789,
+      "sha256": "6c028d9dbf90bea9dcd7daf8c4f529b149deaaabb780edcd764d391783865263"
+    },
+    "evaluation/manual-mac.summary.json": {
+      "bytes": 19604,
+      "sha256": "ecb96cfa02aa55a196e55b3bb9bbb331729d5cc8d0a79b6dd5bfbba958cef75b"
+    },
+    "evaluation/test-mac.jsonl": {
+      "bytes": 794897,
+      "sha256": "6006020aefb90eecd3fa282cf3fceb445e32fc27bfe11bd967ce8cd7a86702d8"
+    },
+    "evaluation/test-mac.summary.json": {
+      "bytes": 38589,
+      "sha256": "17b685763d7a20d3d3256a6264eacf3f569e461cf46eb68ee9160e04ecbe8817"
+    },
+    "evaluation/validation.summary.json": {
+      "bytes": 38081,
+      "sha256": "86a6a6e36522decadd3e93be9bcdc9435c9f619fd16290cfa78542001358d354"
+    },
+    "examples/context-mysql.json": {
+      "bytes": 1366,
+      "sha256": "49fca0302ceb7e5199997a905c309533768cc4e0310be62b092d40dfe19d3209"
+    },
+    "examples/context-supabase.json": {
+      "bytes": 1027,
+      "sha256": "d54c9e69716303ce5a57d79b75f8f26499c6fb6131663ff168e9d5697d9122ea"
+    },
+    "grounding-stats.json": {
+      "bytes": 71,
+      "sha256": "307e4a794c8257fd4fab4040994070560fd3f60a8f73143a7a044c986ea4d545"
+    },
+    "model.safetensors": {
+      "bytes": 279483882,
+      "sha256": "9489b5cde69c11f64b3e7031182e8248aa83790f1ad4aeefbf99a4e5e152e692"
+    },
+    "provenance/experiment-provenance.json": {
+      "bytes": 5894,
+      "sha256": "490b8844864ba2d8116ff898ec7a010ca31f40e900863a317f4b452dcdf2d836"
+    },
+    "provenance/full-audit.json": {
+      "bytes": 363,
+      "sha256": "2616ce8641587452aafaf88f730bd4a9546c5cd5fdd18097e3d2bf9e6c42dec6"
+    },
+    "provenance/native-reference-validation.json": {
+      "bytes": 11624,
+      "sha256": "41289ff92629e67bc77f5e1790cc6e638254d0ab516fd42246fe520e228bb766"
+    },
+    "provenance/selection.json": {
+      "bytes": 552,
+      "sha256": "87972e47a26951aed4e2ac350da3a72c2fc054e1e70649c16f06ff270d724e27"
+    },
+    "provenance/teacher-runtime.json": {
+      "bytes": 2104,
+      "sha256": "d208c7bacc2974395585615eb5baaac0c8f4663fb34f5a8bb40dfa777e01921d"
+    },
+    "provenance/template-filter.json": {
+      "bytes": 1576,
+      "sha256": "f0e55ba2ce0805e04ec3369e4a4a35bc9ff006d10367753b23502e34805635b4"
+    },
+    "requirements.txt": {
+      "bytes": 188,
+      "sha256": "4d2963280263d14196c338b3a20d098d6d5853a70c410504ba717c8b81decaf1"
+    },
+    "runtime/README.md": {
+      "bytes": 1775,
+      "sha256": "a4f8b516d2da87f9c6313ca258680e418572a1ce607ee8d7c847950159498204"
+    },
+    "runtime/download-teacher.py": {
+      "bytes": 193,
+      "sha256": "0f6138e4d46bac757bccbc376cdcb5d88e85c6f5c259412c83db59b4c598afc6"
+    },
+    "runtime/mac-environment.lock.txt": {
+      "bytes": 540,
+      "sha256": "a0196b5cf9b10aa8dfdc7311a534ae032c8647fbcc5c0bb23477097878f0a6f7"
+    },
+    "runtime/requirements-mac.txt": {
+      "bytes": 188,
+      "sha256": "4d2963280263d14196c338b3a20d098d6d5853a70c410504ba717c8b81decaf1"
+    },
+    "runtime/requirements-teacher-cuda.txt": {
+      "bytes": 72,
+      "sha256": "cac0f3bfd6d461da8415ace38468217839cf8185b9ae52a9e3247adca3f834e6"
+    },
+    "runtime/requirements-training-cuda.txt": {
+      "bytes": 299,
+      "sha256": "7b0c3222210c260c9528a94828344ac59405acefdc8118bbf22b652d1dfa792d"
+    },
+    "runtime/serve-teacher.sh": {
+      "bytes": 686,
+      "sha256": "21971a0062a8ede7df97fc6930003fbacb4011729bbfb8ea19b82cecee1daa2b"
+    },
+    "split-audit.json": {
+      "bytes": 1423,
+      "sha256": "5d4275fb3fc9f13e1b9d8b6e6a4f90783db607f43a3d4f886a81b332dc3cdf50"
+    },
+    "tinyquery/__init__.py": {
+      "bytes": 80,
+      "sha256": "83d8a6e71e625d98eef78ce40796f522c4be10aa149deab3bc2c1849b6414ebc"
+    },
+    "tinyquery/audit_data.py": {
+      "bytes": 3526,
+      "sha256": "b7bf574f8f1ea4c34de3b80c0fc98fd98272161c13ce82dafe70c4bea1bd9948"
+    },
+    "tinyquery/average_checkpoints.py": {
+      "bytes": 2239,
+      "sha256": "ff2fe4c30cf7b881e651a91457920a6133b3f72db0f66c3e809b6beb83bfcc99"
+    },
+    "tinyquery/baseline.py": {
+      "bytes": 5342,
+      "sha256": "645c48aad3a492288881cd4f87fd554fa7e19b9c04eee80464f42324a0f8a0a8"
+    },
+    "tinyquery/chat.py": {
+      "bytes": 3706,
+      "sha256": "6afc8a67312f5a909964465656107432a899eca5e59eab0af8bbc68bbb9c5e58"
+    },
+    "tinyquery/checkpoint_eval.py": {
+      "bytes": 2441,
+      "sha256": "b09e552ec0d4af7c892d581798baaf0b6dcb4c379dd13e5c66b73edd1ce44f0d"
+    },
+    "tinyquery/concrete.py": {
+      "bytes": 4755,
+      "sha256": "b4f122bac7f1798dbdae3d1cbef53905435c2f5e76ed0a7761182e7330212780"
+    },
+    "tinyquery/curriculum_data.py": {
+      "bytes": 5910,
+      "sha256": "b5eddaa3c392b275a8c23a65fc8364bb0e48b2207776b0e14ed46e0af1abcdf8"
+    },
+    "tinyquery/data.py": {
+      "bytes": 15996,
+      "sha256": "4f406ea28f33eea0dbff006269cf5f7f2128567be73d1a13b94c3116a3003928"
+    },
+    "tinyquery/evaluate.py": {
+      "bytes": 9364,
+      "sha256": "7bebc27747fc6b3d3ce79471decb42b0dd200c7c7404c366b2c621dd76568204"
+    },
+    "tinyquery/export.py": {
+      "bytes": 2871,
+      "sha256": "cb179b6eecab9f8f1788b5825910414a4e05a786064ea20d13921031b82d7698"
+    },
+    "tinyquery/filter_templates.py": {
+      "bytes": 2130,
+      "sha256": "42adf094bd0d87b8457ec93e72e65e22684530b8a185468ef5a62e118e57221a"
+    },
+    "tinyquery/grounding_data.py": {
+      "bytes": 5630,
+      "sha256": "0c936853adf0bb21dd3709d12622c4a2a5a18fba3fbfcd00f41a77a7cd7bb3fa"
+    },
+    "tinyquery/layout_data.py": {
+      "bytes": 8159,
+      "sha256": "de5cb05aaf6846a37cd0803ff41846f20d968cb8ccf030f6906789a7be95429a"
+    },
+    "tinyquery/manual_eval.py": {
+      "bytes": 5385,
+      "sha256": "7fb830a9d8780e43b3253a71a593296e6f6c8bb27d18f5b717e01ce851d52edf"
+    },
+    "tinyquery/model.py": {
+      "bytes": 9287,
+      "sha256": "6d2f02f31228a010176719f91e113f7c16870c7d4f1cf1f63218d530caccdc43"
+    },
+    "tinyquery/native_sql.py": {
+      "bytes": 5910,
+      "sha256": "0343c466f19b59c4d86bae9dcccf676d247b86bb268d72e38f49e385fc22fd71"
+    },
+    "tinyquery/package_dataset.py": {
+      "bytes": 2998,
+      "sha256": "46d40adb4b810e31aa228f532a88790b501784e8f65e058f2c81805f7d9c32de"
+    },
+    "tinyquery/prepare.py": {
+      "bytes": 3427,
+      "sha256": "28dbce0678889c787e30ae348019b8dd05c41a479f76b8f720c6cacce78b890f"
+    },
+    "tinyquery/recipes.py": {
+      "bytes": 3534,
+      "sha256": "90cb3c82dbc7f50fc9e7a3ffbef8ff1ca4bf011e92389fa0a63e189a7eef0fdc"
+    },
+    "tinyquery/requirements-inference.txt": {
+      "bytes": 188,
+      "sha256": "4d2963280263d14196c338b3a20d098d6d5853a70c410504ba717c8b81decaf1"
+    },
+    "tinyquery/requirements.txt": {
+      "bytes": 299,
+      "sha256": "7b0c3222210c260c9528a94828344ac59405acefdc8118bbf22b652d1dfa792d"
+    },
+    "tinyquery/rescore.py": {
+      "bytes": 1797,
+      "sha256": "01a451361650e10a0992819599da0054162e0c927438a856f086584ed2166ba6"
+    },
+    "tinyquery/scope_data.py": {
+      "bytes": 3056,
+      "sha256": "9596541b871b33682d9a22ab28949aa8f6809d4d25b7fb3e494b033795b0b039"
+    },
+    "tinyquery/teacher_bench.py": {
+      "bytes": 2235,
+      "sha256": "29b9943bc93060ad1cc5fbe91961b1219e87e0333f08d93d1af6ee4d64004635"
+    },
+    "tinyquery/teacher_templates.py": {
+      "bytes": 4131,
+      "sha256": "9c81deae8851da2767fb160720ea16ea8969a7a9094079260bd55a17827a25e2"
+    },
+    "tinyquery/test_adapter.py": {
+      "bytes": 3207,
+      "sha256": "78093a8b4339a2cb39dec4132749ac6de019a834af7c9b4bdc24929e6f8ebe4c"
+    },
+    "tinyquery/test_model.py": {
+      "bytes": 1990,
+      "sha256": "c118a42cc1e590878aa6728279460e87f9f52e1ae9035c48915450e424661d9a"
+    },
+    "tinyquery/train.py": {
+      "bytes": 11493,
+      "sha256": "4529ec9bd24158ed58557d4e1ecb2698ee171c36db6d344c8c67785795ac45f8"
+    },
+    "tinyquery/verify_concrete.py": {
+      "bytes": 3874,
+      "sha256": "701d706c6b7de9409408b727b176d1b2b7e0dae6f263c4aa432fe93a7ed34126"
+    },
+    "tokenization.json": {
+      "bytes": 992,
+      "sha256": "f300d44db96f49fdc7a8ee30f67b8aaa39524023d93a4c045333906c8ee0c5af"
+    },
+    "tokenizer.json": {
+      "bytes": 264747,
+      "sha256": "6e7ce36ebb3345785916cabccaeb6fff24335b3e88f831894ed8ef263396ed1b"
+    }
+  }
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9489b5cde69c11f64b3e7031182e8248aa83790f1ad4aeefbf99a4e5e152e692
+size 279483882

provenance/experiment-provenance.json

@@ -0,0 +1,143 @@
+{
+  "experiment_start_utc": "2026-09-10T10:57:23Z",
+  "hard_deadline_utc": "2026-09-10T14:57:23Z",
+  "student_initialization": "All weights originate from random initialization during this experiment. Copy-head parameters were added later; no pretrained student weights were used.",
+  "teacher_role": "Language templates, direct paraphrases, and round-trip verification; reference SQL semantics constructed programmatically.",
+  "curriculum": [
+    {
+      "data": "v1",
+      "examples": 85777,
+      "tokens": 22739579,
+      "purpose": "Initial semantic and language learning"
+    },
+    {
+      "data": "v2",
+      "examples": 152667,
+      "tokens": 43182146,
+      "purpose": "Random character identifiers and tool names"
+    },
+    {
+      "data": "v3",
+      "examples": 218022,
+      "tokens": 62519864,
+      "purpose": "Training-vocabulary chunks and varied MCP naming styles"
+    },
+    {
+      "data": "v4",
+      "examples": 283377,
+      "tokens": 81750566,
+      "purpose": "Identifiers without fixed prefixes; copies restricted to context/question",
+      "sample_weights": {
+        "latest_65355_variants": 3,
+        "other_rows": 1
+      }
+    },
+    {
+      "data": "v5",
+      "examples": 302893,
+      "tokens": 87335945,
+      "purpose": "Exact copying of project IDs and schema discovery table names",
+      "sample_weights": {
+        "scope_and_discovery_variants": 4,
+        "unprefixed_identifier_variants": 3,
+        "other_rows": 1
+      },
+      "validation_selection": "All 1,200 validation records, token-weighted loss"
+    },
+    {
+      "data": "v6",
+      "examples": 320115,
+      "purpose": "New schema families with related table/parent/project identifiers; authored comparison contrasts; conservative training-template audit filtering",
+      "new_scenarios_before_filter": 6000,
+      "new_rows_before_filter": 24000,
+      "excluded_rows": 6778,
+      "validation_selection": "All 1,200 validation records, token-weighted response loss",
+      "tokens": 92098292,
+      "response_tokens": 12137844
+    },
+    {
+      "data": "v7",
+      "purpose": "Variable schema layouts, random qualified tool names, authored natural city requests and Hindi lexical mappings",
+      "new_layout_variants": 8826,
+      "new_runtime_name_variants": 13635,
+      "new_city_language_rows": 4800,
+      "new_city_scenario_families": 1200,
+      "new_variant_sample_weight": 6,
+      "development_observation": "An informal default-schema demo failed on a shorter schema and natural language. This stage addresses that development failure; the demo is not a held-out benchmark.",
+      "examples": 347376,
+      "tokens": 100224712,
+      "response_tokens": 13163249
+    }
+  ],
+  "implementation_commits": [
+    "f24d869",
+    "c1ce778",
+    "c9e8d77",
+    "f6f0b7b",
+    "780db6a",
+    "4875531",
+    "f713a08",
+    "69e19ff",
+    "1f798c7",
+    "47267b7",
+    "5d7a622",
+    "3ebf37a",
+    "eecb176",
+    "1c50583",
+    "364753b",
+    "a8f336f",
+    "1049d88",
+    "9729c63"
+  ],
+  "architecture_reference": "https://aclanthology.org/P17-1099/",
+  "research_claim": "Established decoder components and a learned pointer-generator adaptation. No claim of a new research architecture or controlled ablation.",
+  "template_audit": {
+    "teacher": "Qwen/Qwen3.8-27B-FP8",
+    "mtp": 3,
+    "jobs": 330,
+    "template_instances": 1313,
+    "accepted": 1276,
+    "flagged": 37,
+    "training_rows_excluded": 6778,
+    "scope": "Training SQL templates only; held-out validation/test/manual phrases excluded",
+    "caveat": "Earlier stages used pre-filter corpus; final continuation uses filtered data"
+  },
+  "manual_split": "Additional hand-written phrasing templates rendered on 40 test scenario families; no shared train/validation scenarios. Manual and test are not independent schema-family samples.",
+  "development_split": "192 schema/tool-name perturbations of 140 validation families; no training, test or manual families. Used for development, not an independent test.",
+  "selection": {
+    "final_rule": "Among frozen evaluated candidates, maximize the mean of full validation task success and development task success; use validation response loss to select additional candidate snapshots. Final test and manual results are read only after weights are selected.",
+    "rationale": "Standard validation alone missed the schema-layout failure found in the development demo."
+  },
+  "refinement": {
+    "data": "v7, unchanged",
+    "start_utc": "2026-09-10T13:40:00Z",
+    "planned_minutes": 18,
+    "batch": 128,
+    "learning_rate": 0.0001,
+    "prompt_weight": 0,
+    "response_weight": 1,
+    "action_auxiliary_weight": 0.05,
+    "rationale": "Focus late optimization on output actions after earlier language learning; reduce unpredictable-input LM gradients. Selection can retain an earlier checkpoint if refinement does not improve development scores."
+  },
+  "selected_checkpoint": {
+    "step": 25715,
+    "sha256": "9489b5cde69c11f64b3e7031182e8248aa83790f1ad4aeefbf99a4e5e152e692",
+    "frozen_at_utc": "2026-09-10T14:00:36.513800+00:00",
+    "test_and_manual_excluded_from_selection": true
+  },
+  "selected_checkpoint_counters": {
+    "step": 25715,
+    "processed_tokens": 595067301,
+    "response_tokens": 78367410,
+    "training_seconds": 5092.33953666687,
+    "random_initialization": true
+  },
+  "completed_trainer_counters": {
+    "step": 32642,
+    "processed_tokens": 851198143,
+    "response_tokens": 113051294,
+    "training_seconds": 7099.562133073807,
+    "note": "Retained-lineage trainer counters include validation/checkpoint time, exclude discarded work/model loading. Last checkpoint was not selected."
+  },
+  "release_archive_limitation": "Vast.ai SSH refused connections after final evaluation completed; local selected weights and datasets were already secured. Some raw remote logs and final optimizer state were not retrieved."
+}

provenance/full-audit.json

@@ -0,0 +1,13 @@
+{
+  "rows": 347376,
+  "unique_ids": 347376,
+  "unique_prompts": 347376,
+  "sql_rows": 262468,
+  "unique_sql_cases": 151920,
+  "distinct_context_sql_compilations": 160253,
+  "fixtures_per_sql_case": 2,
+  "sql_engine": "SQLite after dialect parsing/adaptation; native coverage reported separately",
+  "seconds": 97.74559593200684,
+  "errors": [],
+  "passed": true
+}

provenance/native-reference-validation.json

@@ -0,0 +1,474 @@
+{
+  "seconds": 0.32275891304016113,
+  "cases": 66,
+  "gold_executes": 66,
+  "prediction_equivalent": null,
+  "engine_versions": {
+    "mysql": "8.0.46",
+    "postgresql": "16.15"
+  },
+  "results": [
+    {
+      "id": "c4df2d6a92784c925f92_en",
+      "backend": "supabase",
+      "operation": "all",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "4996e2672571480e6c60_en",
+      "backend": "mysql",
+      "operation": "project",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "dbb8cd8445dd19a458e2_en",
+      "backend": "supabase",
+      "operation": "eq",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "39205adac55e98cd6749_en",
+      "backend": "supabase",
+      "operation": "project_eq",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "5fc77c83422c0e74572f_en",
+      "backend": "supabase",
+      "operation": "gt",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "4217b243f42c7af06169_en",
+      "backend": "mysql",
+      "operation": "lt",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "13ac9d307d6b101b8705_en",
+      "backend": "mysql",
+      "operation": "gte",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "467e53c90a42cd9935a3_en",
+      "backend": "mysql",
+      "operation": "lte",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "e0b7adca29f0eb32fe2c_en",
+      "backend": "mysql",
+      "operation": "between",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "1667f375dfed0ad934a4_en",
+      "backend": "mysql",
+      "operation": "and",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "328b77dd66cdf8cabfd8_en",
+      "backend": "mysql",
+      "operation": "or",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "5c1ca9682dbf9c4df2a1_en",
+      "backend": "supabase",
+      "operation": "null",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "7dfb47d110d38ff48e06_en",
+      "backend": "mysql",
+      "operation": "not_null",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "33d0ab3caa0fba0e3985_en",
+      "backend": "mysql",
+      "operation": "count",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "ad6bf3f875b067153767_en",
+      "backend": "supabase",
+      "operation": "count_eq",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "bc42ac714674eb7e9508_en",
+      "backend": "supabase",
+      "operation": "sum",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "5815b9336617abb93d33_en",
+      "backend": "supabase",
+      "operation": "avg",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "3487e6b7bf3881dd7e68_en",
+      "backend": "mysql",
+      "operation": "max",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "f39ec09a8da50e72bdd5_en",
+      "backend": "supabase",
+      "operation": "min",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "bc6f054aa3be632d1bce_en",
+      "backend": "mysql",
+      "operation": "distinct",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "dae8c9c830c197ee8e74_en",
+      "backend": "mysql",
+      "operation": "sort_desc",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "640719c646d2ad1c92e6_en",
+      "backend": "mysql",
+      "operation": "sort_asc",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "dc2e72d9a61704c6679c_en",
+      "backend": "mysql",
+      "operation": "top",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "9b0921e93df3d40ae856_en",
+      "backend": "supabase",
+      "operation": "bottom",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "c23d51b5c96bb4b300bf_en",
+      "backend": "supabase",
+      "operation": "group_count",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "dafbcd52face40a7b678_en",
+      "backend": "mysql",
+      "operation": "group_sum",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "f551f3aa4afc6f91a2d5_en",
+      "backend": "supabase",
+      "operation": "having",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "f9c2698db9e7dbe6ec8e_en",
+      "backend": "supabase",
+      "operation": "date_after",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "6016d266fd2535bc4479_en",
+      "backend": "supabase",
+      "operation": "year",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "ccbdc5b0dcc9fcc00608_en",
+      "backend": "supabase",
+      "operation": "month",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "068560b98b4de1eee24e_en",
+      "backend": "supabase",
+      "operation": "contains",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "7cf5c995d18de3fe3679_en",
+      "backend": "mysql",
+      "operation": "join",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "f0a44c49f89e568f5a97_en",
+      "backend": "supabase",
+      "operation": "join_filter",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "2d72fda52b91ee1560ab_en",
+      "backend": "mysql",
+      "operation": "all",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "029cd7c058965ed14b05_en",
+      "backend": "mysql",
+      "operation": "eq",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "327bced9ccce8adafb67_en",
+      "backend": "supabase",
+      "operation": "lt",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "c5a56e24a5878c81791a_en",
+      "backend": "supabase",
+      "operation": "gte",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "9925b01d3b059dd22a13_en",
+      "backend": "supabase",
+      "operation": "or",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "5ec7a0dca8e2804dbca9_en",
+      "backend": "supabase",
+      "operation": "not_null",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "8488c3423e53e10c7853_en",
+      "backend": "supabase",
+      "operation": "count",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "7839648878bf9feda17b_en",
+      "backend": "mysql",
+      "operation": "count_eq",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "e956931ef5531d79954c_en",
+      "backend": "mysql",
+      "operation": "avg",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "d2ac91503b4c4b8c4bbf_en",
+      "backend": "supabase",
+      "operation": "max",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "e27bab397ec7daf2a101_en",
+      "backend": "supabase",
+      "operation": "distinct",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "9912a7207dd6e403b2d5_en",
+      "backend": "supabase",
+      "operation": "top",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "90eda6200c21cebc9077_en",
+      "backend": "mysql",
+      "operation": "group_count",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "fe5044da918260a617a9_en",
+      "backend": "supabase",
+      "operation": "group_sum",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "2daba35926021b3697eb_en",
+      "backend": "mysql",
+      "operation": "date_after",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "9992776e0fbabf4da80e_en",
+      "backend": "mysql",
+      "operation": "month",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "b654c9f6e4c53883151c_en",
+      "backend": "supabase",
+      "operation": "join",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "3cfa2682031af21eedd8_en",
+      "backend": "supabase",
+      "operation": "project",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "403211bcb94202245467_en",
+      "backend": "mysql",
+      "operation": "project_eq",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "9ef4d4f5d5bcf5858087_en",
+      "backend": "supabase",
+      "operation": "and",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "8c437f27ac8f48915f9a_en",
+      "backend": "mysql",
+      "operation": "min",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "4c485cb9781970855140_en",
+      "backend": "supabase",
+      "operation": "sort_desc",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "0bdc9f428b0896d7e38a_en",
+      "backend": "supabase",
+      "operation": "sort_asc",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "c4a83c9634139691ddde_en",
+      "backend": "mysql",
+      "operation": "having",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "3b0a2346d3ae970242c1_en",
+      "backend": "mysql",
+      "operation": "year",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "cd68dcf59fb97c84247d_en",
+      "backend": "mysql",
+      "operation": "contains",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "6569be8d680d87bb283d_en",
+      "backend": "mysql",
+      "operation": "gt",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "69aa7071e087609fac06_en",
+      "backend": "supabase",
+      "operation": "lte",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "49dc5daf9217fe08b1c4_en",
+      "backend": "mysql",
+      "operation": "sum",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "f637ccd41642ab190d04_en",
+      "backend": "mysql",
+      "operation": "null",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "0aec7748ac00fd2195f1_en",
+      "backend": "mysql",
+      "operation": "join_filter",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "56594767710f98b9b347_en",
+      "backend": "supabase",
+      "operation": "between",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    },
+    {
+      "id": "eb4c2f1cc33712f0b82e_en",
+      "backend": "mysql",
+      "operation": "bottom",
+      "gold_executes": true,
+      "prediction_equivalent": null
+    }
+  ]
+}

provenance/selection.json

@@ -0,0 +1,19 @@
+{
+  "checkpoint": "/workspace/tinyquery/candidates/step-25715/model.safetensors",
+  "step": 25715,
+  "checkpoint_sha256": "9489b5cde69c11f64b3e7031182e8248aa83790f1ad4aeefbf99a4e5e152e692",
+  "development_scores": {
+    "validation": {
+      "correct": 1174,
+      "total": 1200,
+      "rate": 0.9783333333333334
+    },
+    "development": {
+      "correct": 188,
+      "total": 192,
+      "rate": 0.9791666666666666
+    }
+  },
+  "selection_score": 0.97875,
+  "rule": "Equal mean of full validation and development task success. Test/manual excluded."
+}

provenance/teacher-runtime.json

@@ -0,0 +1,81 @@
+{
+  "teacher": "Qwen/Qwen3.8-27B-FP8",
+  "revision": "017b9c7af6b5689d5dd426a76e0bc077eb5ca20a",
+  "server": {
+    "vllm": "0.28.0",
+    "hardware": "RTX PRO 6000 Blackwell Max-Q Workstation Edition, 96 GB",
+    "context": 8192,
+    "max_num_seqs": 32,
+    "gpu_memory_utilization": 0.85,
+    "language_model_only": true,
+    "fp8_backend": "CUTLASS"
+  },
+  "phases": [
+    {
+      "name": "language_templates",
+      "requests": 45,
+      "concurrency": 16,
+      "temperature": 0.8,
+      "top_p": 0.95,
+      "top_k": 20,
+      "thinking": false,
+      "mtp": false,
+      "max_tokens": 4096
+    },
+    {
+      "name": "direct_paraphrases",
+      "requests": 1500,
+      "concurrency": 32,
+      "temperature": 0.7,
+      "top_p": 0.95,
+      "top_k": 20,
+      "thinking": false,
+      "mtp": false,
+      "max_tokens": 500
+    },
+    {
+      "name": "round_trip_verification",
+      "requests": 1500,
+      "temperature": 0,
+      "thinking": false,
+      "mtp": true,
+      "num_speculative_tokens": 3,
+      "max_tokens": 700
+    },
+    {
+      "name": "training_template_semantic_audit",
+      "requests": 330,
+      "template_instances": 1313,
+      "concurrency": 32,
+      "temperature": 0,
+      "thinking": false,
+      "mtp": true,
+      "num_speculative_tokens": 3,
+      "max_tokens": 700,
+      "accepted_instances": 1276,
+      "flagged_instances": 37
+    }
+  ],
+  "matched_benchmark": {
+    "concurrency": 32,
+    "mtp_off_tps": [
+      688.80098,
+      693.58296
+    ],
+    "mtp_3_tps": [
+      926.42637,
+      935.17359
+    ],
+    "warmup": "one full request round per configuration before measured rounds",
+    "measured_rounds_per_configuration": 2,
+    "requests_per_round": 64,
+    "valid_json_per_configuration": 128,
+    "truncated_per_configuration": 0,
+    "temperature": 0.7,
+    "top_p": 0.8,
+    "top_k": 20,
+    "presence_penalty": 0,
+    "repetition_penalty": 1.0
+  },
+  "scope": "MTP speeds inference; it does not certify semantic correctness. Initial template/paraphrase generation preceded MTP; verification used MTP. No claim of a globally optimal configuration."
+}

provenance/template-filter.json

@@ -0,0 +1,48 @@
+{
+  "audit_jobs": 330,
+  "accepted_template_instances": 1276,
+  "flagged_template_variants": 37,
+  "excluded_training_rows": 6778,
+  "remaining_training_rows": 320115,
+  "exclusions": {
+    "bottom/hi/2": 215,
+    "contains/noisy_en/9": 196,
+    "contains/hi/4": 145,
+    "contains/hinglish/7": 195,
+    "bottom/noisy_en/2": 208,
+    "contains/en/9": 185,
+    "contains/hi/7": 168,
+    "contains/hinglish/8": 202,
+    "lt/hinglish/4": 144,
+    "not_null/hi/5": 196,
+    "top/en/3": 216,
+    "bottom/en/2": 179,
+    "contains/noisy_en/1": 164,
+    "contains/hi/2": 221,
+    "lt/en/4": 190,
+    "not_null/hinglish/5": 164,
+    "contains/en/1": 200,
+    "contains/hi/9": 219,
+    "contains/hinglish/2": 168,
+    "contains/en/7": 187,
+    "contains/hinglish/9": 180,
+    "bottom/hinglish/2": 172,
+    "contains/en/2": 213,
+    "contains/noisy_en/8": 217,
+    "contains/hi/1": 147,
+    "contains/en/4": 219,
+    "contains/noisy_en/2": 176,
+    "max/en/9": 151,
+    "lt/en/5": 161,
+    "contains/hi/8": 175,
+    "not_null/en/5": 172,
+    "contains/noisy_en/4": 228,
+    "contains/en/8": 122,
+    "contains/hinglish/1": 168,
+    "contains/noisy_en/7": 189,
+    "contains/hinglish/4": 189,
+    "lt/hi/4": 137
+  },
+  "policy": "Conservative exclusion after SQL round-trip disagreement; disagreement alone does not prove the teacher was correct. Validation, test, manual, direct verified paraphrases and separately authored contrasts remain unchanged.",
+  "limitation": "Earlier curriculum stages trained on these rows before the audit; this filtering does not undo earlier exposure."
+}

requirements.txt

@@ -0,0 +1,8 @@
+# Tested with PyTorch 2.13.0 on CUDA and 2.14.0 on macOS.
+torch>=2.13,<3
+numpy>=2.2,<3
+tokenizers==0.22.2
+safetensors>=0.8,<1
+sqlglot==30.18.0
+jsonschema>=4.26,<5
+huggingface-hub>=1.28,<2

runtime/README.md

@@ -0,0 +1,16 @@
+# Runtime and reproducibility
+
+The student runs locally on Apple Silicon MPS, CUDA, or CPU. Use Python 3.12 and install `requirements-mac.txt` (or the root `requirements.txt`). `mac-environment.lock.txt` records the actual local environment. CUDA training and teacher requirements belong in separate GPU environments; vLLM FP8 teacher serving is not a Mac runtime.
+
+`serve-teacher.sh` reconstructs the recorded working launch settings. The exact downloaded teacher revision is pinned in `download-teacher.py`. CUDA training used PyTorch 2.13.0+cu130; use the appropriate official CUDA wheel index. The teacher used vLLM 0.28.0 and Transformers 5.15.1. Do not install the CUDA environment into the Mac inference environment.
+
+The published dataset freezes the final corpus; original templates, prompts, verification records and programmatic generators are included in the companion dataset and source package. Training evolved through seven curricula; a fresh run on the final corpus is a new experiment, not a bitwise replay. Earlier training encountered 6,778 rows later filtered out. The released weights are selected by validation and development success, not the last trainer checkpoint. The final optimizer state and some raw remote logs could not be retrieved after the rented SSH endpoint refused connections.
+
+Example fresh training from the published final corpus:
+
+```sh
+python -m tinyquery.prepare --data path/to/extracted-jsonl-splits
+python -m tinyquery.train --data path/to/extracted-jsonl-splits --out runs/new --copy-dim 128 --minutes 120
+```
+
+For the exact dataset tokenizer, pass the existing tokenizer through the prepare script's documented reuse option (`python -m tinyquery.prepare --help`). A new randomly initialized run can obtain different results.

runtime/download-teacher.py

@@ -0,0 +1,2 @@
+from huggingface_hub import snapshot_download
+snapshot_download("Qwen/Qwen3.8-27B-FP8", revision="017b9c7af6b5689d5dd426a76e0bc077eb5ca20a", local_dir="teacher/Qwen3.8-27B-FP8", max_workers=8)

runtime/mac-environment.lock.txt

@@ -0,0 +1,32 @@
+anyio==4.15.1
+attrs==26.1.0
+certifi==2026.7.22
+click==8.5.0
+filelock==3.32.6
+fsspec==2026.7.0
+h11==0.16.0
+hf-xet==1.6.0
+httpcore==1.0.9
+httpx==0.28.1
+huggingface-hub==1.30.0
+idna==3.19
+jinja2==3.1.6
+jsonschema==4.26.0
+jsonschema-specifications==2025.9.1
+markupsafe==3.0.3
+mpmath==1.3.0
+networkx==3.6.1
+numpy==2.5.3
+packaging==26.3
+pyarrow==25.0.1
+pyyaml==6.0.3
+referencing==0.37.0
+rpds-py==2026.6.3
+safetensors==0.8.0
+setuptools==84.0.0
+sqlglot==30.18.0
+sympy==1.14.0
+tokenizers==0.23.2
+torch==2.14.0
+tqdm==4.70.0
+typing-extensions==4.16.0

runtime/requirements-mac.txt

@@ -0,0 +1,8 @@
+# Tested with PyTorch 2.13.0 on CUDA and 2.14.0 on macOS.
+torch>=2.13,<3
+numpy>=2.2,<3
+tokenizers==0.22.2
+safetensors>=0.8,<1
+sqlglot==30.18.0
+jsonschema>=4.26,<5
+huggingface-hub>=1.28,<2

runtime/requirements-teacher-cuda.txt

@@ -0,0 +1,4 @@
+vllm==0.28.0
+torch==2.13.0
+transformers==5.15.1
+huggingface-hub==1.28.0

runtime/requirements-training-cuda.txt

@@ -0,0 +1,11 @@
+# Training environment used on the rented CUDA instance; torch includes its CUDA runtime.
+torch==2.13.0
+numpy==2.2.6
+tokenizers==0.22.2
+safetensors==0.8.0
+sqlglot==30.18.0
+jsonschema==4.26.0
+huggingface-hub==1.28.0
+# Only needed for native database evaluation:
+psycopg[binary]==3.3.5
+pymysql==1.2.0

runtime/serve-teacher.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -euo pipefail
+# Reconstructed from recorded working settings; requires the rented CUDA GPU.
+# Supply a local pinned snapshot path, or download the recorded revision first.
+: "${TEACHER_MODEL_PATH:?Set TEACHER_MODEL_PATH to the downloaded Qwen snapshot}"
+exec vllm serve "$TEACHER_MODEL_PATH" \
+  --served-model-name Qwen/Qwen3.8-27B-FP8 \
+  --host 127.0.0.1 --port 18000 --tensor-parallel-size 1 \
+  --language-model-only --max-model-len 8192 --max-num-seqs 32 \
+  --gpu-memory-utilization 0.85 --reasoning-parser qwen3 \
+  --compilation-config '{"cudagraph_capture_sizes":[1,2,4,8,16,32]}' \
+  --speculative-config '{"method":"mtp","num_speculative_tokens":3}'

split-audit.json

@@ -0,0 +1,76 @@
+{
+  "splits": {
+    "train": {
+      "rows": 347376,
+      "unique_prompts": 347376,
+      "scenario_groups": 25145,
+      "unique_ids": 347376
+    },
+    "validation": {
+      "rows": 1200,
+      "unique_prompts": 1200,
+      "scenario_groups": 300,
+      "unique_ids": 1200
+    },
+    "test": {
+      "rows": 1196,
+      "unique_prompts": 1196,
+      "scenario_groups": 299,
+      "unique_ids": 1196
+    },
+    "manual": {
+      "rows": 160,
+      "unique_prompts": 160,
+      "scenario_groups": 40,
+      "unique_ids": 160
+    },
+    "development": {
+      "rows": 192,
+      "unique_prompts": 192,
+      "scenario_groups": 140,
+      "unique_ids": 192
+    }
+  },
+  "overlap": {
+    "train-validation": {
+      "prompts": 0,
+      "scenarios": 0
+    },
+    "train-test": {
+      "prompts": 0,
+      "scenarios": 0
+    },
+    "train-manual": {
+      "prompts": 0,
+      "scenarios": 0
+    },
+    "train-development": {
+      "prompts": 0,
+      "scenarios": 0
+    },
+    "validation-test": {
+      "prompts": 0,
+      "scenarios": 0
+    },
+    "validation-manual": {
+      "prompts": 0,
+      "scenarios": 0
+    },
+    "validation-development": {
+      "prompts": 0,
+      "scenarios": 140
+    },
+    "test-manual": {
+      "prompts": 0,
+      "scenarios": 40
+    },
+    "test-development": {
+      "prompts": 0,
+      "scenarios": 0
+    },
+    "manual-development": {
+      "prompts": 0,
+      "scenarios": 0
+    }
+  }
+}

tinyquery/__init__.py

@@ -0,0 +1 @@
+"""TinyQuery: an explicitly bounded tool-calling model trained from scratch."""

tinyquery/audit_data.py

@@ -0,0 +1,59 @@
+"""Audit every reference action and execute each distinct SQL case on two fixtures."""
+import argparse
+import hashlib
+import json
+import sqlite3
+from pathlib import Path
+import time
+from jsonschema import Draft202012Validator
+from tinyquery.data import SQL_OPS,compact,serialize,validate_sql,sqlite_sql
+from tinyquery.evaluate import check_action
+
+
+def main():
+    p=argparse.ArgumentParser();p.add_argument('--data',required=True);p.add_argument('--out',required=True)
+    args=p.parse_args();start=time.time();ids=set();prompts=set();queries=set();context_queries=set();validators={};errors=[];count=0;sql_rows=0
+    for line in Path(args.data).open():
+        row=json.loads(line);count+=1
+        try:
+            assert row['id'] not in ids,'Duplicate ID'
+            ids.add(row['id']);fingerprint=hashlib.sha256(row['prompt'].encode()).hexdigest()
+            assert fingerprint not in prompts,'Duplicate prompt'
+            prompts.add(fingerprint)
+            assert row['prompt']==serialize(row['context'],row['question']),'Prompt serialization differs'
+            assert row['response']==compact(row['target']),'Response serialization differs'
+            action=row['target']
+            if action['action']=='call':
+                assert set(action)=={'action','name','arguments'}
+                tool=next(t for t in row['context']['tools'] if t['name']==action['name'])
+                schema=compact(tool['inputSchema'])
+                if schema not in validators:validators[schema]=Draft202012Validator(tool['inputSchema'])
+                validators[schema].validate(action['arguments'])
+            else:check_action(action,row['context'])
+            if row['operation'] in SQL_OPS:
+                sql_rows+=1;a=action['arguments'];sql=a.get('sql',a.get('query'))
+                context_key=hashlib.sha256(compact([row['backend'],row['context']['schema'],sql]).encode()).hexdigest()
+                if context_key not in context_queries:
+                    import sqlglot
+                    db=sqlite3.connect(':memory:');db.create_function('YEAR',1,lambda x:0);db.create_function('MONTH',1,lambda x:0)
+                    try:
+                        for ddl in row['context']['schema']:db.execute(ddl)
+                        db.execute(sqlite_sql(sqlglot.parse_one(sql,read='postgres' if row['backend']=='supabase' else 'mysql')))
+                    finally:db.close()
+                    context_queries.add(context_key)
+                key=hashlib.sha256(compact([row['backend'],row['slots'],sql]).encode()).hexdigest()
+                if key not in queries:
+                    validate_sql(sql,row['backend'],row['slots']);queries.add(key)
+        except Exception as exc:
+            errors.append({'id':row['id'],'error':type(exc).__name__+': '+str(exc)})
+            if len(errors)>=20:break
+        if count%10000==0:print(json.dumps({'rows':count,'unique_sql':len(queries),'seconds':time.time()-start}),flush=True)
+    report={'rows':count,'unique_ids':len(ids),'unique_prompts':len(prompts),'sql_rows':sql_rows,'unique_sql_cases':len(queries),
+            'distinct_context_sql_compilations':len(context_queries),
+            'fixtures_per_sql_case':2,'sql_engine':'SQLite after dialect parsing/adaptation; native coverage reported separately',
+            'seconds':time.time()-start,'errors':errors,'passed':not errors}
+    Path(args.out).write_text(json.dumps(report,indent=2));print(json.dumps(report),flush=True)
+    if errors:raise SystemExit(1)
+
+
+if __name__=='__main__':main()

tinyquery/average_checkpoints.py

@@ -0,0 +1,33 @@
+"""Average compatible checkpoints from the same randomly initialized training lineage."""
+import argparse
+import json
+from pathlib import Path
+import torch
+from safetensors.torch import load_file,save_file
+from tinyquery.prepare import file_sha256
+
+
+def main():
+    p=argparse.ArgumentParser();p.add_argument('--checkpoints',nargs='+',required=True);p.add_argument('--weights',nargs='+',type=float)
+    p.add_argument('--out',required=True);args=p.parse_args();weights=args.weights or [1]*len(args.checkpoints)
+    assert len(weights)==len(args.checkpoints) and all(w>=0 for w in weights) and sum(weights)>0
+    weights=[w/sum(weights) for w in weights];total={};config=None;sources=[]
+    for name,weight in zip(args.checkpoints,weights):
+        path=Path(name);c=json.loads((path.parent/'config.json').read_text());assert config is None or config==c;config=c
+        info=json.loads((path.parent/'checkpoint-info.json').read_text());assert info['random_initialization']
+        state=load_file(str(path));assert not total or total.keys()==state.keys()
+        for key,value in state.items():
+            if key not in total:total[key]=value.float()*weight
+            else:total[key].add_(value.float(),alpha=weight)
+        sources.append({'checkpoint':str(path),'sha256':file_sha256(path),'weight':weight,'info':info});del state
+    dest=Path(args.out);dest.mkdir(parents=True,exist_ok=True)
+    save_file({k:v.to(torch.bfloat16).contiguous() for k,v in total.items()},str(dest/'model.safetensors'),
+              metadata={'method':'weighted_parameter_average','random_initialization':'true'})
+    (dest/'config.json').write_text(json.dumps(config,indent=2))
+    info={'method':'weighted_parameter_average','sources':sources,'random_initialization':True,
+          'step':max(s['info']['step'] for s in sources),'step_interpretation':'Latest source step; these are averaged weights, not that raw checkpoint.'}
+    for key in ['processed_tokens','response_tokens','training_seconds']:info[key]=max(s['info'][key] for s in sources)
+    (dest/'checkpoint-info.json').write_text(json.dumps(info,indent=2));print(json.dumps({'out':str(dest),'sources':[(s['info']['step'],s['weight']) for s in sources]}))
+
+
+if __name__=='__main__':main()

tinyquery/baseline.py

@@ -0,0 +1,90 @@
+"""Transparent TF-IDF nearest-question baseline; copies the retrieved action unchanged."""
+import argparse
+from collections import Counter,defaultdict
+import json
+import math
+from pathlib import Path
+import re
+import time
+import numpy as np
+from tinyquery.evaluate import score,aggregate
+
+
+def bind_context(text,source,destination):
+    """Bind by tool descriptions and DDL positions, using no expected answer or slots."""
+    import sqlglot
+    from sqlglot import exp
+    action=json.loads(text)
+    if action.get('action')!='call':return text
+    old=next(t for t in source['tools'] if t['name']==action['name'])
+    normalize=lambda s:s.replace('PostgreSQL','SQL').replace('MySQL','SQL')
+    candidates=[t for t in destination['tools'] if normalize(t['description'])==normalize(old['description'])]
+    if not candidates:return text
+    tool=candidates[0];props=tool['inputSchema']['properties'];args=dict(action['arguments'])
+    sqlkey=next((k for k in ['sql','query'] if isinstance(args.get(k),str) and args[k].startswith('SELECT ')),None)
+    if sqlkey:
+        targetkey=next((k for k in ['sql','query'] if k in props),sqlkey)
+        tables={};columns={}
+        for old_ddl,new_ddl in zip(source['schema'],destination['schema']):
+            a=sqlglot.parse_one(old_ddl).this;b=sqlglot.parse_one(new_ddl).this
+            tables[a.this.name]=b.this.name
+            for c,d in zip(a.expressions,b.expressions):
+                if isinstance(c,exp.ColumnDef) and isinstance(d,exp.ColumnDef):columns[c.name]=d.name
+        query=sqlglot.parse_one(args.pop(sqlkey),read='postgres' if source['backend']=='supabase' else 'mysql')
+        def rename(node):
+            if isinstance(node,exp.Table) and node.name in tables:node.set('this',exp.to_identifier(tables[node.name]))
+            if isinstance(node,exp.Column):
+                if node.name in columns:node.set('this',exp.to_identifier(columns[node.name]))
+                if node.table in tables:node.set('table',exp.to_identifier(tables[node.table]))
+            return node
+        args[targetkey]=query.transform(rename).sql(dialect='postgres' if destination['backend']=='supabase' else 'mysql')+';'
+    args={k:v for k,v in args.items() if k in props}
+    if 'project_id' in props:args['project_id']=destination['project_id']
+    return json.dumps({'action':'call','name':tool['name'],'arguments':args},ensure_ascii=False,separators=(',',':'))
+
+
+def words(text):
+    return re.findall(r'[^\W_]+|_',text.lower(),re.UNICODE)
+
+
+def main():
+    p=argparse.ArgumentParser(); p.add_argument('--train',required=True); p.add_argument('--data',required=True)
+    p.add_argument('--out',required=True);p.add_argument('--bind-context',action='store_true')
+    args=p.parse_args(); start=time.time()
+    training=[]; seen=set(); counts=[]; df=Counter()
+    for line in Path(args.train).open():
+        row=json.loads(line)
+        if row['question'] in seen: continue
+        seen.add(row['question']); terms=Counter(words(row['question']))
+        training.append((row['id'],row['response'],row['context'] if args.bind_context else None)); counts.append(terms); df.update(terms.keys())
+    n=len(training); idf={word:math.log((1+n)/(1+freq))+1 for word,freq in df.items()}
+    postings=defaultdict(list)
+    for i,terms in enumerate(counts):
+        weighted={word:(1+math.log(freq))*idf[word] for word,freq in terms.items()}
+        norm=math.sqrt(sum(v*v for v in weighted.values())) or 1
+        for word,value in weighted.items(): postings[word].append((i,value/norm))
+    postings={word:(np.array([i for i,_ in pairs]),np.array([v for _,v in pairs],dtype=np.float32))
+              for word,pairs in postings.items()}
+    results=[]; groups=defaultdict(list); out=Path(args.out); out.parent.mkdir(parents=True,exist_ok=True)
+    with out.open('w') as stream:
+        for line in Path(args.data).open():
+            row=json.loads(line); similarities=np.zeros(n,dtype=np.float32)
+            for word,freq in Counter(words(row['question'])).items():
+                if word in postings:
+                    indices,weights=postings[word]; similarities[indices]+=(1+math.log(freq))*idf[word]*weights
+            index=int(similarities.argmax()); source,text,context=training[index]
+            if args.bind_context:
+                try:text=bind_context(text,context,row['context'])
+                except (ValueError,KeyError,StopIteration,AttributeError):pass
+            metrics=score(row,text)
+            results.append(metrics)
+            for field in ['language','backend','operation']: groups[field+':'+row[field]].append(metrics)
+            stream.write(json.dumps({'id':row['id'],'retrieved_id':source,'output':text,'metrics':metrics},ensure_ascii=False)+'\n')
+    report={'baseline':'TF-IDF nearest training question; copy action unchanged, no tool/schema adaptation',
+            'training_questions':n,'examples':len(results),'seconds':time.time()-start,'metrics':aggregate(results),
+            'groups':{key:aggregate(value) for key,value in groups.items()}}
+    if args.bind_context:report['baseline']='TF-IDF nearest question plus tool-description and DDL-position binding; no expected answer or gold slots used'
+    out.with_suffix('.summary.json').write_text(json.dumps(report,indent=2)); print(json.dumps(report['metrics']))
+
+
+if __name__=='__main__': main()

tinyquery/chat.py

@@ -0,0 +1,68 @@
+"""Stream the student's raw JSON response from a checkpoint; optionally render an MCP request."""
+import argparse
+import json
+import sys
+import time
+from pathlib import Path
+import torch
+from tokenizers import Tokenizer,decoders
+from tinyquery.data import serialize,make_tools
+from tinyquery.evaluate import load_model,check_action,mcp_request,parse_action
+
+
+def default_context(backend):
+    import random
+    tools,_,_,_=make_tools(backend,random.Random(4),'train','demo')
+    return {'backend':backend,'project_id':'demo',
+            'schema':['CREATE TABLE customers (id INTEGER PRIMARY KEY, name TEXT, city TEXT, amount REAL);'],
+            'tools':tools,'policy':'Read-only database access. Use provided tools. Ask when required information is missing.'}
+
+
+def stream_response(model,tokenizer,prompt,max_tokens=160,stats=None):
+    ids=tokenizer.encode(prompt).ids
+    if len(ids)+max_tokens>model.config.context: raise ValueError('Prompt plus output budget exceeds model context')
+    device=next(model.parameters()).device; tokens=torch.tensor([ids],device=device)
+    past=None; decoder=decoders.DecodeStream(skip_special_tokens=True)
+    generated=[]; emitted='';start=time.perf_counter();first_token=None
+    with torch.inference_mode():
+        for _ in range(max_tokens):
+            logits,past,_=model(tokens,past=past,use_cache=True,last_only=True)
+            token=int(logits[0,-1].argmax()); generated.append(token)
+            if first_token is None:first_token=time.perf_counter()-start
+            text=decoder.step(tokenizer,token)
+            if text: emitted+=text; yield text
+            if token==tokenizer.token_to_id('<|end|>'): break
+            tokens=torch.tensor([[token]],device=device)
+    full=tokenizer.decode(generated,skip_special_tokens=True)
+    if full.startswith(emitted) and len(full)>len(emitted): yield full[len(emitted):]
+    if stats is not None:
+        seconds=time.perf_counter()-start
+        stats.update(generated_tokens=len(generated),seconds=seconds,tokens_per_second=len(generated)/seconds,
+                     first_token_seconds=first_token,device=str(device),includes_model_loading=False)
+
+
+def main():
+    p=argparse.ArgumentParser(); p.add_argument('question'); p.add_argument('--checkpoint',required=True)
+    p.add_argument('--tokenizer'); p.add_argument('--context',help='JSON file with backend, schema and MCP-style tools')
+    p.add_argument('--backend',choices=['mysql','supabase'],default='supabase')
+    p.add_argument('--tokens',type=int,default=160); p.add_argument('--mcp',action='store_true')
+    p.add_argument('--stats',action='store_true',help='Print measured generation speed to stderr after streaming')
+    args=p.parse_args(); device='cuda' if torch.cuda.is_available() else ('mps' if torch.backends.mps.is_available() else 'cpu')
+    torch.set_num_threads(4)
+    tokenizer=Tokenizer.from_file(args.tokenizer or str(Path(args.checkpoint).parent/'tokenizer.json'))
+    model=load_model(args.checkpoint,device)
+    context=json.loads(Path(args.context).read_text()) if args.context else default_context(args.backend)
+    answer='';stats={}
+    for text in stream_response(model,tokenizer,serialize(context,args.question),args.tokens,stats):
+        print(text,end='',flush=True); answer+=text
+    print(flush=True)
+    if args.stats:print(json.dumps(stats),file=sys.stderr)
+    try:
+        action=check_action(parse_action(answer),context)
+        if args.mcp and action['action']=='call': print(json.dumps(mcp_request(action,context),ensure_ascii=False,indent=2))
+    except Exception as exc:
+        print('Output validation failed:',str(exc),file=sys.stderr)
+        raise SystemExit(1)
+
+
+if __name__=='__main__': main()

tinyquery/checkpoint_eval.py

@@ -0,0 +1,38 @@
+"""Freeze an owned training checkpoint and evaluate only development splits."""
+import argparse
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import torch
+from safetensors.torch import save_file
+
+
+def main():
+    p=argparse.ArgumentParser();p.add_argument('--checkpoint',required=True);p.add_argument('--data',required=True)
+    p.add_argument('--out',required=True);args=p.parse_args();data=Path(args.data)
+    r=torch.load(args.checkpoint,map_location='cpu',weights_only=False)
+    dest=Path(args.out)/('step-'+str(r['step']));dest.mkdir(parents=True,exist_ok=True)
+    if (dest/'selection.json').exists():print((dest/'selection.json').read_text());return
+    save_file({k:v.to(torch.bfloat16).contiguous() for k,v in r['model'].items()},str(dest/'model.safetensors'),
+              metadata={'step':str(r['step']),'random_initialization':'true'})
+    (dest/'config.json').write_text(json.dumps(r['config'],indent=2))
+    info={k:r[k] for k in ['step','processed_tokens','response_tokens','training_seconds','random_initialization']}
+    (dest/'checkpoint-info.json').write_text(json.dumps(info,indent=2));del r
+    print(json.dumps({'frozen':str(dest),'step':info['step']}),flush=True);scores={};digest=None
+    for split in ['validation','development']:
+        if not (data/(split+'.jsonl')).exists():continue
+        with (dest/(split+'.log')).open('w') as log:
+            subprocess.run([sys.executable,'-u','-m','tinyquery.evaluate','--checkpoint',str(dest/'model.safetensors'),
+                            '--tokenizer',str(data/'tokenizer.json'),'--data',str(data/(split+'.jsonl')),
+                            '--out',str(dest/(split+'.jsonl'))],stdout=log,stderr=subprocess.STDOUT,check=True)
+        summary=json.loads((dest/(split+'.summary.json')).read_text());scores[split]=summary['metrics']['success']
+        digest=summary['checkpoint_sha256'];print(json.dumps({'split':split,'metrics':summary['metrics']}),flush=True)
+    selection={'checkpoint':str(dest/'model.safetensors'),'step':info['step'],'checkpoint_sha256':digest,
+               'development_scores':scores,'selection_score':sum(v['rate'] for v in scores.values())/len(scores),
+               'rule':'Equal mean of full validation and development task success. Test/manual excluded.'}
+    (dest/'selection.json').write_text(json.dumps(selection,indent=2));print(json.dumps(selection),flush=True)
+
+
+if __name__=='__main__':main()

tinyquery/concrete.py

@@ -0,0 +1,78 @@
+"""Directly paraphrase concrete training scenarios, keeping the original reference action."""
+import argparse
+import concurrent.futures
+import json
+import random
+import re
+import time
+import urllib.request
+from pathlib import Path
+from tinyquery.data import compact,serialize
+from tinyquery.recipes import LANGUAGES
+
+
+def paraphrase(row,base,seed):
+    task={'schema':row['context']['schema'],'backend':row['backend'],'reference_request':row['question'],
+          'reference_action':row['target']}
+    prompt=('Write one natural user request in each of four styles that has exactly the reference meaning. '
+            'Use conversational phrasing, not instructions about writing SQL. Keep every numeric bound, '
+            'literal value, selected field, sorting and limit unchanged. Do not add a new condition. '
+            'You may use ordinary English/Hindi words for obvious schema fields, but preserve database '
+            'literal values exactly. Output JSON keys en, noisy_en, hi, hinglish with string values. '
+            'hi must be Devanagari Hindi, hinglish Romanized Hindi, noisy_en imperfect English. Task: '+compact(task))
+    payload={'model':'Qwen/Qwen3.8-27B-FP8','messages':[{'role':'user','content':prompt}],
+             'max_tokens':500,'temperature':.7,'seed':seed,'response_format':{'type':'json_object'},
+             'chat_template_kwargs':{'enable_thinking':False}}
+    request=urllib.request.Request(base+'/chat/completions',data=compact(payload).encode(),headers={'Content-Type':'application/json'})
+    with urllib.request.urlopen(request,timeout=180) as response: result=json.load(response)
+    parsed=json.loads(result['choices'][0]['message']['content'])
+    accepted=[]
+    for lang in LANGUAGES:
+        question=parsed.get(lang)
+        if not isinstance(question,str) or not 5<len(question)<900 or '{' in question: continue
+        if lang=='hi' and not re.search('[\u0900-\u097f]',question): continue
+        if re.search('[\u0600-\u06ff]',question): continue
+        # Reject changed numeric bounds and quoted database literals in constrained SQL recipes.
+        source_numbers=set(re.findall(r'(?<![a-zA-Z_])\d+(?:\.\d+)?',row['question']))
+        output_numbers=set(re.findall(r'(?<![a-zA-Z_])\d+(?:\.\d+)?',question))
+        if source_numbers!=output_numbers: continue
+        required={'eq':['value'],'project_eq':['value'],'and':['value'],'or':['value','other'],
+                  'count_eq':['value'],'contains':['value'],'join_filter':['value']}.get(row['operation'],[])
+        if any(str(row['slots'][k]).casefold() not in question.casefold() for k in required): continue
+        new=dict(row); new['id']=row['scenario_id']+'_concrete_'+lang; new['language']=lang
+        new['question']=question; new['prompt']=serialize(new['context'],question)
+        new['provenance']='Qwen3.8-27B-FP8 direct concrete paraphrase; numeric/literal checks; semantic audit separate'
+        accepted.append(new)
+    return {'scenario_id':row['scenario_id'],'request':payload,'raw_response':result,'accepted':accepted}
+
+
+def main():
+    p=argparse.ArgumentParser(); p.add_argument('--data',required=True); p.add_argument('--out',required=True)
+    p.add_argument('--count',type=int,default=2500); p.add_argument('--workers',type=int,default=32)
+    p.add_argument('--base',default='http://127.0.0.1:18000/v1'); args=p.parse_args()
+    from tinyquery.data import SQL_OPS
+    seen=set(); candidates=[]
+    with open(args.data) as stream:
+        for line in stream:
+            r=json.loads(line)
+            if r['scenario_id'] not in seen and r['operation'] in SQL_OPS:
+                seen.add(r['scenario_id']); candidates.append(r)
+    random.Random(887).shuffle(candidates); candidates=candidates[:args.count]
+    out=Path(args.out); out.parent.mkdir(parents=True,exist_ok=True)
+    done=set()
+    if out.exists():
+        done={json.loads(line)['scenario_id'] for line in out.read_text().splitlines()}
+    start=time.time(); total=0; completed=0
+    with out.open('a') as stream, concurrent.futures.ThreadPoolExecutor(args.workers) as pool:
+        jobs={pool.submit(paraphrase,r,args.base,2100+i):r['scenario_id'] for i,r in enumerate(candidates) if r['scenario_id'] not in done}
+        for future in concurrent.futures.as_completed(jobs):
+            completed+=1
+            try:
+                result=future.result(); stream.write(compact(result)+'\n'); stream.flush()
+                total+=len(result['accepted'])
+            except Exception as exc:
+                print(compact({'error':str(exc),'scenario_id':jobs[future]}),flush=True)
+            if completed%25==0: print(compact({'completed':completed,'accepted':total,'seconds':time.time()-start}),flush=True)
+
+
+if __name__=='__main__': main()