diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..f938136e3adacfd92be087f6e113b5d6d97f678f --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2026 Alibaba Cloud + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..63a14fba4268b8267b6c60a8b51ad3c087f06e90 --- /dev/null +++ b/README.md @@ -0,0 +1,111 @@ +--- +language: +- en +- hi +license: apache-2.0 +library_name: pytorch +tags: +- text-to-sql +- tool-calling +- mcp +- from-scratch +- tiny-model +- custom-code +- safetensors +datasets: +- karmx/TinyQuery-Tools-Multilingual +--- + +# TinyQuery-140M + +**139.7M parameters, trained from random initialization on one RTX PRO 6000 within a four-hour experiment.** TinyQuery turns a question, database schema and runtime tool definitions into a JSON action. It targets English, imperfect English, Hindi and Hinglish, with bounded read-only MySQL and PostgreSQL/Supabase queries and MCP-style tool calls. + +The frozen student achieved **92.47% full task success (1,106/1,196)** on the held-out test and **82.50% (132/160)** on additional handwritten-phrasing cases. It is an experimental narrow model, not a general chatbot or a guarantee of correct SQL. + +This repository contains the weights, training/inference source, tokenizer, configurations, evaluation reports, architecture documentation and runtime recipes. **This is a custom native PyTorch checkpoint. Transformers `AutoModel` and `pipeline()` cannot load it.** + +## Run with streaming output + +Requires Python 3.12. Download this repository, then run from its directory: + +```sh +python3.12 -m venv .venv +source .venv/bin/activate +python -m pip install -r requirements.txt +python -m tinyquery.chat "दिल्ली के ग्राहकों के नाम दिखाओ।" \ + --checkpoint model.safetensors --backend supabase --mcp --stats +``` + +The CLI automatically uses Apple Silicon MPS, CUDA or CPU and streams raw output. The bundled fictitious demo schema supports this example: + +```json +{"action":"call","name":"execute_sql","arguments":{"query":"SELECT name FROM customers WHERE city = 'Delhi';"}} +``` + +Use `--context examples/context-supabase.json` or `examples/context-mysql.json`, edited with your schema and exact tool definitions. A context has `backend`, `project_id`, `schema`, `tools` and `policy`; tools use `name`, `description` and JSON Schema `inputSchema`. The context limit is 2,048 tokens including the output budget; final training sequences reached 945 tokens. + +`--mcp` validates and serializes a JSON-RPC `tools/call` payload. **It does not connect to an MCP server, authenticate, or execute a database query.** The host application must implement transport and execution. The structural validator does not prove semantic correctness. Invalid output causes a nonzero CLI exit status. + +A single local M5/MPS demonstration measured approximately 52 generated tokens/second, excluding model loading. This is one short run, not a throughput guarantee. + +## Evaluation + +Weights were frozen at step 25,715 on 2026-09-10 at 14:00:36 UTC, before reading test/manual results. Selection maximized the equal mean of full validation and development success among completed evaluated candidates. Development contains schema-layout and tool-name perturbations of validation families. + +| Measure | Held-out test | Additional manual phrasing | +|---|---:|---:| +| Examples | 1,196 | 160 | +| Valid JSON | 1,196/1,196 (100%) | 160/160 (100%) | +| Valid action/tool schema | 1,191/1,196 (99.58%) | 159/160 (99.38%) | +| Correct tool, call cases | 1,088/1,100 (98.91%) | 146/152 (96.05%) | +| Exact arguments, call cases | 1,014/1,100 (92.18%) | 124/152 (81.58%) | +| SQL result equivalence, SQL cases | 896/912 (98.25%) | 94/112 (83.93%) | +| **Full task success** | **1,106/1,196 (92.47%)** | **132/160 (82.50%)** | +| TF-IDF retrieval + context-binding baseline, full success | 881/1,196 (73.66%) | 115/160 (71.88%) | + +These primary numbers are the completed CUDA BF16 evaluation captured in `evaluation/gpu-final-console-metrics.json`. The rented server subsequently refused SSH connections, so its raw final prediction files were not retrieved. Independent FP32 Mac evaluation reproduced every aggregate metric exactly; complete raw predictions, language/backend breakdowns and summaries are included in `evaluation/test-mac.*` and `evaluation/manual-mac.*`. No retraining or selection follows the test results. + +Evaluation uses raw greedy generation with no repair, constrained decoding, retrieval fallback or teacher fallback. Full SQL-task success requires the correct tool, exact non-SQL arguments (including project scope), compilation against the supplied schema, and matching results on two generated SQLite fixtures after dialect adaptation. SQL equivalence alone does not require correct project scope, so its percentage can exceed full success. Non-SQL tool calls require exact arguments; clarification/answer text requires exact reference wording. + +**These are synthetic benchmark results, not measured production task success.** Native MySQL 8.0.46/PostgreSQL 16.15 checks passed all 66 reference-operation/dialect cases; these validate reference data, not the final student's test predictions. Final native student evaluation was unavailable after SSH access failed. SQL fixture equivalence is not a proof for all possible databases. + +Validation scored 1,174/1,200 (97.83%); development scored 188/192 (97.92%). These were used for model selection and are not final held-out claims. The 160 manual cases use handwritten phrasing templates on 40 test families, so manual and test are not independent schema-family samples. + +## Architecture and training + +| Component | Value | +|---|---| +| Total parameters | 139,738,113 | +| Decoder | 12 layers, width 1,024 | +| Attention | 16 query heads, 4 KV heads, head dimension 64 | +| Feed-forward | SwiGLU, intermediate width 2,816 | +| Positions / normalization | RoPE, RMSNorm | +| Embeddings | Tied input/output; train-only byte BPE vocabulary of 4,082 | +| Source-copy head | Learned 128-dimensional pointer projection and mixture gate | +| Auxiliary head | Three action classes, used in training | +| Context | 2,048 | +| Export | BF16 Safetensors, approximately 280 MB | + +The source-copy head mixes vocabulary generation with attention over the supplied context and question, excluding generated answer text. It adapts the established [pointer-generator approach](https://aclanthology.org/P17-1099/). The architecture and dataset evolved together, so this is not a controlled ablation or a claim of a novel research architecture. See [architecture documentation](docs/architecture.md). + +All student weights originate from random initialization during this experiment; the copy head was initialized later and trained with the continuing decoder. No pretrained student checkpoint or teacher logits were used. Training used BF16, AdamW, gradient clipping and weighted length buckets. Early loss weights were response 1, prompt 0.15, and auxiliary action CE 0.05. A later response-only refinement did not displace the selected checkpoint. + +The selected checkpoint records 595,067,301 processed tokens, 78,367,410 response tokens and 5,092.34 trainer seconds. These are retained-lineage counters, including validation/checkpoint time and excluding discarded work and model loading. Training continued to step 32,642, but those last weights were not selected. The overall experiment began at 10:57:23 UTC and had a 14:57:23 UTC deadline. See [experiment provenance](provenance/experiment-provenance.json) for curricula and counters. + +## Dataset and teacher + +The companion [TinyQuery-Tools-Multilingual dataset](https://huggingface.co/datasets/karmx/TinyQuery-Tools-Multilingual) contains 347,376 final training rows, 25,145 scenario groups, 100,224,712 tokens and 13,163,249 response tokens. Language/context variants are not independent teacher generations. SQL/tool semantics are programmatic; Qwen supplies templates, paraphrases and verification. An additional city-language curriculum is programmatically authored and labeled accordingly. + +Teacher: [Qwen/Qwen3.8-27B-FP8](https://huggingface.co/Qwen/Qwen3.8-27B-FP8), revision `017b9c7af6b5689d5dd426a76e0bc077eb5ca20a`. On the rented RTX PRO 6000, vLLM 0.28.0 with MTP-3 measured about 931 output tokens/second versus 691 without MTP at 32 concurrent requests. Initial template/paraphrase generation preceded MTP; verification and template audit used MTP. This is a measured workload comparison, not a claim of globally optimal settings. See [teacher runtime provenance](provenance/teacher-runtime.json) and [runtime recipes](runtime/README.md). + +Training has no scenario-group overlap with validation, development, test or manual. There are no exact prompt duplicates across splits. Development shares validation families, and manual shares test families, intentionally. A teacher audit flagged 37 template instances; 6,778 derived rows were removed from the final corpus after earlier training stages had already encountered them. Final reference-data audits passed schema/serialization and generated-fixture checks; these checks do not certify all natural-language paraphrases. + +## Scope and limitations + +- Supports a bounded recipe set: projections, filters, aggregates, grouping/HAVING, ordering, date/string filtering and a single join, plus schema discovery and a few generic tool categories. +- Arbitrary unseen databases, tools, complex joins, long conversations and broad Hindi translation are not established capabilities. Hindi city/table lexical mappings cover a small explicit training vocabulary. +- A correct JSON object can contain a wrong tool, literal, project ID or query. The model is not 100% reliable. +- It emits one action, not a complete autonomous MCP agent. Database execution and authorization belong to the host application. +- The runtime source and frozen data support new experiments; the release is not a bitwise replay of the evolving training session. The final optimizer state and some server logs were not recovered after SSH access failed. + +Weights and project code are released under Apache-2.0. Teacher provenance is documented separately. See `manifest.json` for file hashes and `checkpoint-info.json` for selected-weight counters. diff --git a/checkpoint-info.json b/checkpoint-info.json new file mode 100644 index 0000000000000000000000000000000000000000..d80496bb7a93b1992edd870edd8780f6d0882544 --- /dev/null +++ b/checkpoint-info.json @@ -0,0 +1 @@ +{"step": 25715, "processed_tokens": 595067301, "response_tokens": 78367410, "training_seconds": 5092.33953666687, "random_initialization": true} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000000000000000000000000000000000000..f3b20216734507cd877284677e4ca9e7e035b1d0 --- /dev/null +++ b/config.json @@ -0,0 +1,11 @@ +{ + "vocab_size": 4082, + "width": 1024, + "layers": 12, + "heads": 16, + "kv_heads": 4, + "hidden": 2816, + "context": 2048, + "rope_theta": 10000.0, + "copy_dim": 128 +} \ No newline at end of file diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000000000000000000000000000000000000..474caf8ccfedac58bc7e02d8b5e09e56e376714e --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,65 @@ +# TinyQuery: how the model works + +TinyQuery is a 139,738,113-parameter decoder trained from random weights for a narrow task: turn a question plus database schema and tool definitions into one JSON action. It uses established transformer components and a learned source-copy head. It is an experiment, not a claim of a new research architecture. + +```mermaid +flowchart LR + A[Schema, tools and question] --> B[Byte BPE: 4,082 tokens] + B --> C[12 causal decoder blocks] + C --> D[Generate next-token probabilities] + C --> E[Copy probabilities over input tokens] + C --> F[Learned sigmoid gate] + D --> G[Mixed next-token distribution] + E --> G + F --> G + G --> H[Raw JSON action] + H --> I[Validate tool name and arguments] + I --> J[MCP tools/call payload] +``` + +The model sees everything needed for a particular request in its prompt. It must learn the language of the request, the required SQL operation, the correct runtime tool, and how to preserve identifiers and literal values. The inference CLI prints the action; it does not connect to a database or execute the tool. + +## Decoder + +| Component | Configuration | +|---|---| +| Tokenizer | Byte BPE trained only on training text; 4,082 tokens | +| Hidden width | 1,024 | +| Decoder layers | 12 | +| Attention | 16 query heads, 4 key/value heads, head width 64 | +| Positions | Rotary position embeddings, theta 10,000 | +| Normalization | RMSNorm before attention and feed-forward layers | +| Feed-forward | SwiGLU, intermediate width 2,816 | +| Output projection | Tied to the input token embedding | +| Context limit | 2,048 tokens; training coverage is shorter and reported with the dataset | +| Training-only auxiliary head | Three action classes: call, clarify, answer | + +Attention is causal: a position can use earlier tokens but cannot see future answer tokens. Grouped-query attention shares four sets of keys and values across sixteen query heads, reducing the KV cache used during streaming. The cache preserves earlier attention keys and values so generation does not recompute the entire prompt for each token. + +## Learned copying + +Ordinary next-token generation initially memorized familiar table and tool names. Dataset variants and a copy head address this specific failure. The copy head projects decoder states into 128-dimensional queries and keys, attends to the supplied context and question, and adds together attention mass for repeated occurrences of the same token. + +For each next token, a learned gate combines two distributions: + +`P(token) = gate × P_generate(token) + (1 − gate) × P_copy_from_input(token)` + +The copy distribution excludes generated answer text. This prevents the model from repeatedly copying its own earlier mistakes. The gate and attention are learned; there is no hard-coded replacement of predicted tool names, no SQL template compiler, no constrained decoding, and no hidden teacher or retrieval fallback in neural evaluation. + +This adapts the established [pointer-generator idea](https://aclanthology.org/P17-1099/). Data and architecture changed together during development, so improvements are not presented as a controlled architecture ablation. + +## Training and output + +Early stages give answer tokens weight 1 and prompt tokens weight 0.15, plus 0.05 times an auxiliary action-classification loss. A late refinement uses prompt weight 0, focusing optimization on answers; final checkpoint selection can retain an earlier candidate if that refinement does not improve development results. The prompt contribution helps learn vocabulary from scratch; the answer contribution teaches the requested behavior. The final curriculum also samples particular context variants more often to counter identifier-format memorization. + +The teacher supplies language variations and a verification pass. Programmatic recipes supply reference SQL and tool actions. Student weights originate entirely from random initialization in this experiment; later stages continue those same weights and add randomly initialized copy parameters. + +The output contract is either a call, a clarification, or a short answer: + +```json +{"action":"call","name":"execute_sql","arguments":{"query":"SELECT name FROM customers WHERE city = 'Delhi';"}} +``` + +The adapter checks the action against the supplied tool schema and can serialize a JSON-RPC `tools/call` request. JSON validity, correct tool selection, exact project scope and SQL result correctness are separate evaluation measures. A valid JSON action alone does not establish a correct query. + +The model's scope is bounded SQL and tool use in four language styles. It is not a general conversational model or a database knowledge store. Final measured results, training time and limitations belong in the release model card. diff --git a/docs/experiment.md b/docs/experiment.md new file mode 100644 index 0000000000000000000000000000000000000000..59897b7cb9f52b501db350df70d563ce18f663fc --- /dev/null +++ b/docs/experiment.md @@ -0,0 +1,59 @@ +# TinyQuery Tools + +An experimental, randomly initialized decoder for schema-conditioned tool calls in English, imperfect English, Hindi and Hinglish. This is separate from the original 4.2M-parameter Tiny English learning project. + +The current student has **139,738,113 parameters**: 12 decoder layers, width 1024, 16 query heads, four KV heads, rotary positions, RMSNorm, SwiGLU width 2816, tied embeddings, a three-class auxiliary action head and a learned 128-dimensional source-copy head. The copy head mixes next-token generation with attention over the supplied context and question; it excludes generated answer text. The byte BPE vocabulary has 4,082 tokens learned from the training corpus. The configured context limit is 2,048; current training sequences reach 945 tokens. These are established transformer components with a custom training objective, not a demonstrated research breakthrough. + +## Output contract + +The model receives a backend, schema, runtime tool definitions and a question, and predicts one JSON action: + +```json +{"action":"call","name":"execute_sql","arguments":{"query":"SELECT name FROM customers WHERE city = 'Delhi';"}} +``` + +Other actions are `{"action":"clarify","question":"..."}` and `{"action":"answer","text":"..."}`. Provided tool definitions use MCP-style `name`, `description`, and `inputSchema`. The adapter validates the action and can serialize a JSON-RPC `tools/call` request. **The neural model does not implement the MCP transport or authenticate to databases.** + +The database corpus targets read-only MySQL and PostgreSQL (Supabase). It covers projections, comparisons, NULLs, counts and aggregates, grouping/HAVING, sorting/limits, date filtering, substring matching and a single join. Schema discovery and bounded tool-error cases are included. Generic examples cover weather lookup, documentation search, file reading and ticket lookup. These are trained categories, not a claim of competence with arbitrary unseen tools. + +## Dataset + +The final continuation corpus contains 347,376 examples and 100,224,712 tokens, including 13,163,249 response tokens. It combines programmatic SQL/tool semantics, Qwen3.8-27B-FP8 multilingual templates and direct paraphrases, and randomized runtime identifiers. It has 25,145 scenario groups; rendered language/context variants are not independent teacher generations. + +Validation has 1,200 examples and test has 1,196, with held-out domain/table names and language templates. An additional 160 cases render hand-written phrasings on 40 test scenario families. These add phrasing coverage, not independent schema families. Neither test nor manual is used for training or model selection. A separate 192-case development split changes schema layouts and tool names on 140 validation families; it is used during development and shares no training/test/manual families. + +A teacher audit of 1,313 training SQL template instances flagged 37 variants; 6,778 derived rows were conservatively removed from the final corpus. Earlier training stages had already used them. Every final training action passed schema/serialization checks, and 151,920 distinct reference SQL cases executed on two generated SQLite fixtures. A separate 66-case reference check passed on native MySQL 8.0.46 and PostgreSQL 16.15. A further 160,253 distinct query/context pairs compile against the supplied schemas. These are reference-data checks, not student accuracy. + +Teacher revision: `017b9c7af6b5689d5dd426a76e0bc077eb5ca20a`. MTP-3 improved the measured 32-concurrent-request teacher workload from about 691 to 931 output tokens/second. Samples retain teacher prompts, settings and verification provenance. The model is trained from random weights; no pretrained student checkpoint or hidden teacher fallback is used. + +## Reproduce + +```sh +python -m tinyquery.teacher_templates --out data/tinyquery/templates.jsonl +python -m tinyquery.data --templates data/tinyquery/templates.jsonl --out data/tinyquery +python -m tinyquery.prepare --data data/tinyquery +python -m tinyquery.train --data data/tinyquery --out runs/tinyquery --copy-dim 128 --minutes 30 +``` + +`concrete.py` and `verify_concrete.py` add direct paraphrases and round-trip verification. The published dataset includes the frozen augmented corpus; see its provenance and manifest rather than assuming the four commands above reproduce the exact published rows. An optional `--deadline` supplies a hard UTC wall-clock stop. The released experiment used multiple curriculum stages and a fixed four-hour overall budget; a new 30-minute run does not reproduce its result. + +## Stream output + +Once the finished export is placed in `runs/tinyquery/`: + +```sh +.venv/bin/python -m tinyquery.chat "दिल्ली के ग्राहकों के नाम दिखाओ।" \ + --checkpoint runs/tinyquery/model.safetensors --backend supabase --mcp +``` + +Use `--context path/to/context.json` to provide actual schema and tool definitions. The CLI streams the model's raw output and validates it. `--stats` reports measured generation speed. The MCP serializer rejects mismatched project scope, unknown schema tables and unsupported data-changing statements; these structural checks do not establish semantic correctness. It does not execute tools. The default context is a small fictitious customers table. + +## Evaluation + +```sh +python -m tinyquery.evaluate --checkpoint runs/tinyquery/model.safetensors \ + --tokenizer runs/tinyquery/tokenizer.json --data data/tinyquery/test.jsonl \ + --out runs/tinyquery/test-predictions.jsonl +``` + +Evaluation uses raw greedy output with no JSON repair, constrained decoding or teacher fallback. Report JSON validity, tool-schema validity, tool choice, exact arguments and SQL result equivalence separately. The default SQL evaluator uses dialect adaptation to SQLite; `native_sql.py` performs separate checks on local MySQL/PostgreSQL engines. Final measured student results and failure examples will be written after training. diff --git a/evaluation/baseline-manual.jsonl b/evaluation/baseline-manual.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..a757038da5d852b5964bdca0b6c37c4178aac82a --- /dev/null +++ b/evaluation/baseline-manual.jsonl @@ -0,0 +1,160 @@ +{"id": "manual_adff4eabe2639c0c128c_en", "retrieved_id": "a1b28dd3b5e89d70dccb_concrete_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_adff4eabe2639c0c128c_noisy_en", "retrieved_id": "755cf7dd59fa3c3634e3_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_adff4eabe2639c0c128c_hi", "retrieved_id": "b7f50817c901971c7f7a_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT description FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_adff4eabe2639c0c128c_hinglish", "retrieved_id": "a008cb35da6057a0324f_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE status = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_efee65268e6dc2bb069e_en", "retrieved_id": "b228aef51c67d0673184_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT customer_name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_efee65268e6dc2bb069e_noisy_en", "retrieved_id": "c91d2572f9472c9cec38_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT customer_name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_efee65268e6dc2bb069e_hi", "retrieved_id": "3aac2f1448f136869091_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT customer_name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_efee65268e6dc2bb069e_hinglish", "retrieved_id": "9f26d63bfba42c873223_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE rating > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_2c31add7d7b45843dec1_en", "retrieved_id": "6f49986bb93bef798e08_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM inspections WHERE kind = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_2c31add7d7b45843dec1_noisy_en", "retrieved_id": "53759e65346adaac1c7b_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM inspections WHERE kind = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_2c31add7d7b45843dec1_hi", "retrieved_id": "cbdf82266732007c41af_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM inspections WHERE kind = 'pending';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_2c31add7d7b45843dec1_hinglish", "retrieved_id": "6f49986bb93bef798e08_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM inspections WHERE kind = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_82c104614df1ce6e2762_en", "retrieved_id": "faee6951f2a9f841f1b7_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE salary > 100;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_82c104614df1ce6e2762_noisy_en", "retrieved_id": "944f318073a2bcd9624d_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE salary > 100;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_82c104614df1ce6e2762_hi", "retrieved_id": "7c6e0d36bbbdd7d0c41a_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE salary > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_82c104614df1ce6e2762_hinglish", "retrieved_id": "c9d4184209742377db82_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT DISTINCT category FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_35406fa024e66bcb5dd7_en", "retrieved_id": "93d7892cb1dbca43d963_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE score < 10000;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_35406fa024e66bcb5dd7_noisy_en", "retrieved_id": "d3e2b6e356dd98075c1f_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE score < 10000;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_35406fa024e66bcb5dd7_hi", "retrieved_id": "89b49bcc926a3e70ce83_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_35406fa024e66bcb5dd7_hinglish", "retrieved_id": "8bc1ddeb33b08c2be641_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE score >= 10000;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_0b8db53d711baed4f4ed_en", "retrieved_id": "ee309d02322a1ea31a21_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE cost > 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_0b8db53d711baed4f4ed_noisy_en", "retrieved_id": "3c390ab6e128eb92a927_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE cost >= 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_0b8db53d711baed4f4ed_hi", "retrieved_id": "5c402e1eb92eb1a39327_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE cost >= 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_0b8db53d711baed4f4ed_hinglish", "retrieved_id": "75239d67882146f98408_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE cost >= 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_b02f421828127e2eb96b_en", "retrieved_id": "b34987da74688618f914_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price <= 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_b02f421828127e2eb96b_noisy_en", "retrieved_id": "f410e7d2e9f171dbfc08_concrete_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM exhibits GROUP BY kind HAVING COUNT(*) > 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_b02f421828127e2eb96b_hi", "retrieved_id": "b34987da74688618f914_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price <= 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_b02f421828127e2eb96b_hinglish", "retrieved_id": "b706c1a087379cbbac6d_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price <= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_2e2ac3c5dcf95429daaa_en", "retrieved_id": "dbb34637fd547075f1d6_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_2e2ac3c5dcf95429daaa_noisy_en", "retrieved_id": "c8fc7444d05f02f427f0_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_2e2ac3c5dcf95429daaa_hi", "retrieved_id": "f7d9a89b39671e7655a1_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_2e2ac3c5dcf95429daaa_hinglish", "retrieved_id": "c8fc7444d05f02f427f0_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_40464b00088c9159e3d0_en", "retrieved_id": "1ba145f4ee3b2aa19fe8_en", "output": "{\"action\":\"call\",\"name\":\"app.get_weather\",\"arguments\":{\"city\":\"Mumbai\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": false, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": false, "success": false, "error": "Unknown tool"}} +{"id": "manual_40464b00088c9159e3d0_noisy_en", "retrieved_id": "b339be824e371ca04217_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT SUM(rating) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_40464b00088c9159e3d0_hi", "retrieved_id": "36057c87ac77846cee49_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT COUNT(*) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_40464b00088c9159e3d0_hinglish", "retrieved_id": "6040c6883947ec5b3821_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM inspections;\",\"project_id\":\"demo_inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_d735c1ac1979ff1e8620_en", "retrieved_id": "caba0067fdd24c562152_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(price) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_d735c1ac1979ff1e8620_noisy_en", "retrieved_id": "caba0067fdd24c562152_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(price) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_d735c1ac1979ff1e8620_hi", "retrieved_id": "207b7dbc231f51af4760_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT kind, SUM(price) FROM inspections GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_d735c1ac1979ff1e8620_hinglish", "retrieved_id": "8703856ba26d23f81d4e_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(price) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_8efcf517aee64807a8e5_en", "retrieved_id": "97a28182b0b973aa9701_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_8efcf517aee64807a8e5_noisy_en", "retrieved_id": "b9936b4bad68a1e50b75_concrete_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_8efcf517aee64807a8e5_hi", "retrieved_id": "e16c4dae0f3afac7f310_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_8efcf517aee64807a8e5_hinglish", "retrieved_id": "75e3910b4be843450062_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_cab55a7429427fbc5093_en", "retrieved_id": "39cfbf14abe75381fce5_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_cab55a7429427fbc5093_noisy_en", "retrieved_id": "fbfc633a68d2a1ee4dc1_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_cab55a7429427fbc5093_hi", "retrieved_id": "fd60f3f12d6b58fc4c63_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_cab55a7429427fbc5093_hinglish", "retrieved_id": "851c98b607598265e5b5_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_b9c176c07809f42564a0_en", "retrieved_id": "89700fa6c178672acbb9_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT MAX(amount) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_b9c176c07809f42564a0_noisy_en", "retrieved_id": "72d5e4666d38219ea72f_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_b9c176c07809f42564a0_hi", "retrieved_id": "1ec2512105f780c64272_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY CASE WHEN amount IS NULL THEN 1 ELSE 0 END DESC, amount DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_b9c176c07809f42564a0_hinglish", "retrieved_id": "c81d8353510c1d7e35d5_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_475287613764159ee5eb_en", "retrieved_id": "27c83f55d9755a9c140a_concrete_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT category, SUM(amount) FROM reservations GROUP BY category;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_475287613764159ee5eb_noisy_en", "retrieved_id": "39ffd26f283fc2242233_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_475287613764159ee5eb_hi", "retrieved_id": "f5157d5860b54b580212_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT category FROM reservations;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_475287613764159ee5eb_hinglish", "retrieved_id": "6dfc3a46fb6c06bcfe9b_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_5a8f2e207d0efe5b018c_en", "retrieved_id": "9f9bcfa7a1b76cdf247b_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"],\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_5a8f2e207d0efe5b018c_noisy_en", "retrieved_id": "eebe3b000d91bc99d054_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"],\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_5a8f2e207d0efe5b018c_hi", "retrieved_id": "d862008c541c7c0edae1_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM specimens;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "manual_5a8f2e207d0efe5b018c_hinglish", "retrieved_id": "fb4e17dc10585daed130_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"],\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_c4210296305105ab7472_en", "retrieved_id": "08dae4c9658c14af9a8b_noisy_en", "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "manual_c4210296305105ab7472_noisy_en", "retrieved_id": "981c50947db171d5fec5_hinglish", "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false, "text_exact": false}} +{"id": "manual_c4210296305105ab7472_hi", "retrieved_id": "55ce86b87b3b3440fc8e_hi", "output": "{\"action\":\"clarify\",\"question\":\"सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "manual_c4210296305105ab7472_hinglish", "retrieved_id": "981c50947db171d5fec5_hinglish", "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "manual_754691bd1eea25d41ccf_en", "retrieved_id": "1173d51478c0f0770b25_en", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_754691bd1eea25d41ccf_noisy_en", "retrieved_id": "1173d51478c0f0770b25_en", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_754691bd1eea25d41ccf_hi", "retrieved_id": "04fd2c1a67c04de23381_hi", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_754691bd1eea25d41ccf_hinglish", "retrieved_id": "1173d51478c0f0770b25_en", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_252c6053295d69a93c94_en", "retrieved_id": "ba47da34289636eb8037_noisy_en", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_252c6053295d69a93c94_noisy_en", "retrieved_id": "3ef00bb2f51a6ed18daa_en", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_252c6053295d69a93c94_hi", "retrieved_id": "61c17ce29d5fca0cef4f_hi", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_252c6053295d69a93c94_hinglish", "retrieved_id": "750a046184db7426f758_noisy_en", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_f73e21e0eb10d6617995_en", "retrieved_id": "03bb9beb503dc3f084f0_en", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_f73e21e0eb10d6617995_noisy_en", "retrieved_id": "8705522869caaa48e039_noisy_en", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_f73e21e0eb10d6617995_hi", "retrieved_id": "8705522869caaa48e039_hi", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_f73e21e0eb10d6617995_hinglish", "retrieved_id": "0e78e9ded20fe5272477_hinglish", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_8a696e3c3522dd69d5ac_en", "retrieved_id": "8f99a3bc5f418131697b_noisy_en", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-1809\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "manual_8a696e3c3522dd69d5ac_noisy_en", "retrieved_id": "8730c789c1fa0e697d4f_hinglish", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-127\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "manual_8a696e3c3522dd69d5ac_hi", "retrieved_id": "city_29306ae64f1590153417_hi_layout_99707", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM contacts WHERE city = 'Delhi';\",\"project_id\":\"demo_inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "manual_8a696e3c3522dd69d5ac_hinglish", "retrieved_id": "8730c789c1fa0e697d4f_hinglish", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-127\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "manual_2ad3e251c936135e2005_en", "retrieved_id": "a1b28dd3b5e89d70dccb_concrete_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_2ad3e251c936135e2005_noisy_en", "retrieved_id": "755cf7dd59fa3c3634e3_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_2ad3e251c936135e2005_hi", "retrieved_id": "b7f50817c901971c7f7a_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_2ad3e251c936135e2005_hinglish", "retrieved_id": "a008cb35da6057a0324f_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE kind = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_a060620c638d2dc3759e_en", "retrieved_id": "48b174ae126a25408b6f_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_a060620c638d2dc3759e_noisy_en", "retrieved_id": "739c908473cacc7eadc6_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_a060620c638d2dc3759e_hi", "retrieved_id": "04c518f7497c48a695a7_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_a060620c638d2dc3759e_hinglish", "retrieved_id": "9f26d63bfba42c873223_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE balance > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_eb982cf071094adaca4f_en", "retrieved_id": "f0fd5b687a47200d08cc_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT name FROM reservations WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_eb982cf071094adaca4f_noisy_en", "retrieved_id": "f0fd5b687a47200d08cc_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT name FROM reservations WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_eb982cf071094adaca4f_hi", "retrieved_id": "cbdf82266732007c41af_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM reservations WHERE category = 'pending';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_eb982cf071094adaca4f_hinglish", "retrieved_id": "f0fd5b687a47200d08cc_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT name FROM reservations WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_c023a5b6f95173ae856d_en", "retrieved_id": "cfd917d72e37aed08030_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE score > 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_c023a5b6f95173ae856d_noisy_en", "retrieved_id": "734a4347b6aef6b54122_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE score > 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_c023a5b6f95173ae856d_hi", "retrieved_id": "8de2a0fe75d4cb6dc3b8_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE score > 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_c023a5b6f95173ae856d_hinglish", "retrieved_id": "c9d4184209742377db82_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT DISTINCT kind FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_affa5e5e14444cd82bb4_en", "retrieved_id": "6a3bbef7eab5a98a8204_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE salary < 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_affa5e5e14444cd82bb4_noisy_en", "retrieved_id": "6a3bbef7eab5a98a8204_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE salary < 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_affa5e5e14444cd82bb4_hi", "retrieved_id": "78d0a9c46fad0d5b9731_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY salary ASC LIMIT 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_affa5e5e14444cd82bb4_hinglish", "retrieved_id": "7ab81eb1923cb8e1c5ea_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE salary < 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_c4f25575abfd24e3cd63_en", "retrieved_id": "44b7c6f0585479334d72_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE cost <= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_c4f25575abfd24e3cd63_noisy_en", "retrieved_id": "6c759f421afd3b87729c_concrete_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT region, COUNT(*) FROM reservations GROUP BY region HAVING COUNT(*) > 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_c4f25575abfd24e3cd63_hi", "retrieved_id": "c64c305ca0ce02c3d563_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE cost >= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_c4f25575abfd24e3cd63_hinglish", "retrieved_id": "b706c1a087379cbbac6d_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE cost <= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_23c0aa3a4596f995ad2a_en", "retrieved_id": "1ba145f4ee3b2aa19fe8_en", "output": "{\"action\":\"call\",\"name\":\"app.get_weather\",\"arguments\":{\"city\":\"Mumbai\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": false, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": false, "success": false, "error": "Unknown tool"}} +{"id": "manual_23c0aa3a4596f995ad2a_noisy_en", "retrieved_id": "b339be824e371ca04217_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT SUM(amount) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_23c0aa3a4596f995ad2a_hi", "retrieved_id": "36057c87ac77846cee49_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_23c0aa3a4596f995ad2a_hinglish", "retrieved_id": "6040c6883947ec5b3821_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_b564e1a7b1e7fc6d2ab4_en", "retrieved_id": "8d7c829657a8468028fe_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(score) FROM specimens;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_b564e1a7b1e7fc6d2ab4_noisy_en", "retrieved_id": "e88c7ea3cba7722735db_concrete_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(score) FROM specimens;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_b564e1a7b1e7fc6d2ab4_hi", "retrieved_id": "c71e53cd4a9240c41192_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(score) FROM specimens;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_b564e1a7b1e7fc6d2ab4_hinglish", "retrieved_id": "6ba6fa33bb36ac7792b9_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(score) FROM specimens;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_2164b376f732a1c371ab_en", "retrieved_id": "27c83f55d9755a9c140a_concrete_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, SUM(score) FROM specimens GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_2164b376f732a1c371ab_noisy_en", "retrieved_id": "0517beaf82464fadc60f_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM specimens GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_2164b376f732a1c371ab_hi", "retrieved_id": "d367c82ffd966970c68f_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT DISTINCT kind FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_2164b376f732a1c371ab_hinglish", "retrieved_id": "3b1bddc042f483280893_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM specimens GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_4e42bbd40467b2b8c27f_en", "retrieved_id": "6821429a91693b8eafb4_noisy_en", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"SQL joins\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_4e42bbd40467b2b8c27f_noisy_en", "retrieved_id": "e1237888e640d85992e3_en", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"SQL joins\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_4e42bbd40467b2b8c27f_hi", "retrieved_id": "33f141708d95562cf305_hi", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"SQL joins\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_4e42bbd40467b2b8c27f_hinglish", "retrieved_id": "a336da01d4ae0dde0687_noisy_en", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"SQL joins\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_e8f90532d34c12209a83_en", "retrieved_id": "fe0c250a5b61d276b6ce_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE salary > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_e8f90532d34c12209a83_noisy_en", "retrieved_id": "980ddf0f65b7700a2fb6_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(salary) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_e8f90532d34c12209a83_hi", "retrieved_id": "df2bcb22f6685fc818a1_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE salary < 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_e8f90532d34c12209a83_hinglish", "retrieved_id": "f20130bbd19fc0c8fd2a_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE salary >= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_04b2e5886e750df09a51_en", "retrieved_id": "348951272c18ab797a78_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE display_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_04b2e5886e750df09a51_noisy_en", "retrieved_id": "c9ac52ed487c1889af4f_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE display_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_04b2e5886e750df09a51_hi", "retrieved_id": "f7d9a89b39671e7655a1_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE display_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_04b2e5886e750df09a51_hinglish", "retrieved_id": "0c268171e2f1d1321d9b_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE display_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_d30264bbc3b08aedbc51_en", "retrieved_id": "4ef6b6c0e983997e2ad0_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT SUM(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_d30264bbc3b08aedbc51_noisy_en", "retrieved_id": "4ef6b6c0e983997e2ad0_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT SUM(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_d30264bbc3b08aedbc51_hi", "retrieved_id": "207b7dbc231f51af4760_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT city, SUM(score) FROM exhibits GROUP BY city;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_d30264bbc3b08aedbc51_hinglish", "retrieved_id": "b36fba3845c7dc8f8585_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT SUM(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_7dd163238936d96d0bd4_en", "retrieved_id": "64b95cbab60bce765622_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT city FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_7dd163238936d96d0bd4_noisy_en", "retrieved_id": "ba18b57d6a751c27e75e_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT city FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_7dd163238936d96d0bd4_hi", "retrieved_id": "026f4d2a18eddb751c30_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT city FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_7dd163238936d96d0bd4_hinglish", "retrieved_id": "851c98b607598265e5b5_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT city FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_fd0bab1cc08d7298f131_en", "retrieved_id": "7cad2ba59a27227cbdd1_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(salary) FROM exhibits;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_fd0bab1cc08d7298f131_noisy_en", "retrieved_id": "331c202e8594d94e7dbc_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY salary ASC LIMIT 20;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_fd0bab1cc08d7298f131_hi", "retrieved_id": "1ec2512105f780c64272_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_fd0bab1cc08d7298f131_hinglish", "retrieved_id": "982a431f770356ef4ffe_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_6f3055c580b33340c854_en", "retrieved_id": "00319aa9372ffea135d7_en", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_6f3055c580b33340c854_noisy_en", "retrieved_id": "24837896562853f7b6a4_noisy_en", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_6f3055c580b33340c854_hi", "retrieved_id": "258c24044187ca1265c5_hi", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_6f3055c580b33340c854_hinglish", "retrieved_id": "d53f5c6036055adc1894_hinglish", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_5ebc52b3d20bde2076b3_en", "retrieved_id": "c745f3dc5489adf68488_en", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6124\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_5ebc52b3d20bde2076b3_noisy_en", "retrieved_id": "c745f3dc5489adf68488_en", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6124\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_5ebc52b3d20bde2076b3_hi", "retrieved_id": "c745f3dc5489adf68488_hi", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6124\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_5ebc52b3d20bde2076b3_hinglish", "retrieved_id": "c745f3dc5489adf68488_en", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6124\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_8ebc9e868ee558a8ddfe_en", "retrieved_id": "08dae4c9658c14af9a8b_noisy_en", "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "manual_8ebc9e868ee558a8ddfe_noisy_en", "retrieved_id": "981c50947db171d5fec5_hinglish", "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false, "text_exact": false}} +{"id": "manual_8ebc9e868ee558a8ddfe_hi", "retrieved_id": "55ce86b87b3b3440fc8e_hi", "output": "{\"action\":\"clarify\",\"question\":\"सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "manual_8ebc9e868ee558a8ddfe_hinglish", "retrieved_id": "981c50947db171d5fec5_hinglish", "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "manual_88b478ce80eed61bdfb4_en", "retrieved_id": "03a2503aa4caeb5b0396_en", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_88b478ce80eed61bdfb4_noisy_en", "retrieved_id": "03a2503aa4caeb5b0396_en", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_88b478ce80eed61bdfb4_hi", "retrieved_id": "fcc2c16620594edcdeae_hi", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_88b478ce80eed61bdfb4_hinglish", "retrieved_id": "03a2503aa4caeb5b0396_en", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_9bf8948becf74e81a103_en", "retrieved_id": "9f9bcfa7a1b76cdf247b_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_9bf8948becf74e81a103_noisy_en", "retrieved_id": "eebe3b000d91bc99d054_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_9bf8948becf74e81a103_hi", "retrieved_id": "d862008c541c7c0edae1_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "manual_9bf8948becf74e81a103_hinglish", "retrieved_id": "fb4e17dc10585daed130_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} diff --git a/evaluation/baseline-manual.summary.json b/evaluation/baseline-manual.summary.json new file mode 100644 index 0000000000000000000000000000000000000000..a562efe7a551a9dee10c0b9b64639c7e7af30ff7 --- /dev/null +++ b/evaluation/baseline-manual.summary.json @@ -0,0 +1,1007 @@ +{ + "baseline": "TF-IDF nearest question plus tool-description and DDL-position binding; no expected answer or gold slots used", + "training_questions": 212073, + "examples": 160, + "seconds": 7.722788095474243, + "metrics": { + "json_valid": { + "correct": 160, + "total": 160, + "rate": 1.0 + }, + "schema_valid": { + "correct": 158, + "total": 160, + "rate": 0.9875 + }, + "action_correct": { + "correct": 160, + "total": 160, + "rate": 1.0 + }, + "tool_correct": { + "correct": 147, + "total": 152, + "rate": 0.9671052631578947 + }, + "arguments_exact": { + "correct": 109, + "total": 152, + "rate": 0.7171052631578947 + }, + "sql_equivalent": { + "correct": 75, + "total": 112, + "rate": 0.6696428571428571 + }, + "success": { + "correct": 115, + "total": 160, + "rate": 0.71875 + } + }, + "groups": { + "language:en": { + "json_valid": { + "correct": 40, + "total": 40, + "rate": 1.0 + }, + "schema_valid": { + "correct": 38, + "total": 40, + "rate": 0.95 + }, + "action_correct": { + "correct": 40, + "total": 40, + "rate": 1.0 + }, + "tool_correct": { + "correct": 36, + "total": 38, + "rate": 0.9473684210526315 + }, + "arguments_exact": { + "correct": 29, + "total": 38, + "rate": 0.7631578947368421 + }, + "sql_equivalent": { + "correct": 20, + "total": 28, + "rate": 0.7142857142857143 + }, + "success": { + "correct": 31, + "total": 40, + "rate": 0.775 + } + }, + "backend:mysql": { + "json_valid": { + "correct": 80, + "total": 80, + "rate": 1.0 + }, + "schema_valid": { + "correct": 79, + "total": 80, + "rate": 0.9875 + }, + "action_correct": { + "correct": 80, + "total": 80, + "rate": 1.0 + }, + "tool_correct": { + "correct": 74, + "total": 76, + "rate": 0.9736842105263158 + }, + "arguments_exact": { + "correct": 57, + "total": 76, + "rate": 0.75 + }, + "sql_equivalent": { + "correct": 38, + "total": 56, + "rate": 0.6785714285714286 + }, + "success": { + "correct": 60, + "total": 80, + "rate": 0.75 + } + }, + "operation:all": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 4, + "total": 8, + "rate": 0.5 + }, + "sql_equivalent": { + "correct": 4, + "total": 8, + "rate": 0.5 + }, + "success": { + "correct": 4, + "total": 8, + "rate": 0.5 + } + }, + "language:noisy_en": { + "json_valid": { + "correct": 40, + "total": 40, + "rate": 1.0 + }, + "schema_valid": { + "correct": 40, + "total": 40, + "rate": 1.0 + }, + "action_correct": { + "correct": 40, + "total": 40, + "rate": 1.0 + }, + "tool_correct": { + "correct": 38, + "total": 38, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 31, + "total": 38, + "rate": 0.8157894736842105 + }, + "sql_equivalent": { + "correct": 22, + "total": 28, + "rate": 0.7857142857142857 + }, + "success": { + "correct": 31, + "total": 40, + "rate": 0.775 + } + }, + "language:hi": { + "json_valid": { + "correct": 40, + "total": 40, + "rate": 1.0 + }, + "schema_valid": { + "correct": 40, + "total": 40, + "rate": 1.0 + }, + "action_correct": { + "correct": 40, + "total": 40, + "rate": 1.0 + }, + "tool_correct": { + "correct": 35, + "total": 38, + "rate": 0.9210526315789473 + }, + "arguments_exact": { + "correct": 21, + "total": 38, + "rate": 0.5526315789473685 + }, + "sql_equivalent": { + "correct": 14, + "total": 28, + "rate": 0.5 + }, + "success": { + "correct": 23, + "total": 40, + "rate": 0.575 + } + }, + "language:hinglish": { + "json_valid": { + "correct": 40, + "total": 40, + "rate": 1.0 + }, + "schema_valid": { + "correct": 40, + "total": 40, + "rate": 1.0 + }, + "action_correct": { + "correct": 40, + "total": 40, + "rate": 1.0 + }, + "tool_correct": { + "correct": 38, + "total": 38, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 38, + "rate": 0.7368421052631579 + }, + "sql_equivalent": { + "correct": 19, + "total": 28, + "rate": 0.6785714285714286 + }, + "success": { + "correct": 30, + "total": 40, + "rate": 0.75 + } + }, + "backend:supabase": { + "json_valid": { + "correct": 80, + "total": 80, + "rate": 1.0 + }, + "schema_valid": { + "correct": 79, + "total": 80, + "rate": 0.9875 + }, + "action_correct": { + "correct": 80, + "total": 80, + "rate": 1.0 + }, + "tool_correct": { + "correct": 73, + "total": 76, + "rate": 0.9605263157894737 + }, + "arguments_exact": { + "correct": 52, + "total": 76, + "rate": 0.6842105263157895 + }, + "sql_equivalent": { + "correct": 37, + "total": 56, + "rate": 0.6607142857142857 + }, + "success": { + "correct": 55, + "total": 80, + "rate": 0.6875 + } + }, + "operation:project": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 6, + "total": 8, + "rate": 0.75 + }, + "sql_equivalent": { + "correct": 6, + "total": 8, + "rate": 0.75 + }, + "success": { + "correct": 6, + "total": 8, + "rate": 0.75 + } + }, + "operation:project_eq": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 6, + "total": 8, + "rate": 0.75 + }, + "sql_equivalent": { + "correct": 6, + "total": 8, + "rate": 0.75 + }, + "success": { + "correct": 6, + "total": 8, + "rate": 0.75 + } + }, + "operation:gt": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 6, + "total": 8, + "rate": 0.75 + }, + "sql_equivalent": { + "correct": 6, + "total": 8, + "rate": 0.75 + }, + "success": { + "correct": 6, + "total": 8, + "rate": 0.75 + } + }, + "operation:lt": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 5, + "total": 8, + "rate": 0.625 + }, + "sql_equivalent": { + "correct": 5, + "total": 8, + "rate": 0.625 + }, + "success": { + "correct": 5, + "total": 8, + "rate": 0.625 + } + }, + "operation:gte": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 4, + "total": 8, + "rate": 0.5 + }, + "sql_equivalent": { + "correct": 4, + "total": 8, + "rate": 0.5 + }, + "success": { + "correct": 4, + "total": 8, + "rate": 0.5 + } + }, + "operation:lte": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 4, + "total": 8, + "rate": 0.5 + }, + "sql_equivalent": { + "correct": 4, + "total": 8, + "rate": 0.5 + }, + "success": { + "correct": 4, + "total": 8, + "rate": 0.5 + } + }, + "operation:null": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "success": { + "correct": 8, + "total": 8, + "rate": 1.0 + } + }, + "operation:count": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 6, + "total": 8, + "rate": 0.75 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 6, + "total": 8, + "rate": 0.75 + }, + "arguments_exact": { + "correct": 4, + "total": 8, + "rate": 0.5 + }, + "sql_equivalent": { + "correct": 4, + "total": 8, + "rate": 0.5 + }, + "success": { + "correct": 4, + "total": 8, + "rate": 0.5 + } + }, + "operation:sum": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 5, + "total": 8, + "rate": 0.625 + }, + "sql_equivalent": { + "correct": 5, + "total": 8, + "rate": 0.625 + }, + "success": { + "correct": 5, + "total": 8, + "rate": 0.625 + } + }, + "operation:avg": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "success": { + "correct": 8, + "total": 8, + "rate": 1.0 + } + }, + "operation:distinct": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "success": { + "correct": 8, + "total": 8, + "rate": 1.0 + } + }, + "operation:top": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 3, + "total": 8, + "rate": 0.375 + }, + "sql_equivalent": { + "correct": 3, + "total": 8, + "rate": 0.375 + }, + "success": { + "correct": 3, + "total": 8, + "rate": 0.375 + } + }, + "operation:group_count": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 4, + "total": 8, + "rate": 0.5 + }, + "sql_equivalent": { + "correct": 4, + "total": 8, + "rate": 0.5 + }, + "success": { + "correct": 4, + "total": 8, + "rate": 0.5 + } + }, + "operation:list_tables": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 6, + "total": 8, + "rate": 0.75 + }, + "arguments_exact": { + "correct": 6, + "total": 8, + "rate": 0.75 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 6, + "total": 8, + "rate": 0.75 + } + }, + "operation:ambiguous": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "arguments_exact": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 6, + "total": 8, + "rate": 0.75 + } + }, + "operation:weather": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 8, + "total": 8, + "rate": 1.0 + } + }, + "operation:search": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 8, + "total": 8, + "rate": 1.0 + } + }, + "operation:read_file": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 8, + "total": 8, + "rate": 1.0 + } + }, + "operation:ticket": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 7, + "total": 8, + "rate": 0.875 + }, + "arguments_exact": { + "correct": 4, + "total": 8, + "rate": 0.5 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 4, + "total": 8, + "rate": 0.5 + } + } + } +} \ No newline at end of file diff --git a/evaluation/baseline-test.jsonl b/evaluation/baseline-test.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..7b011290e67e868f0346e12f005c188eac514c7c --- /dev/null +++ b/evaluation/baseline-test.jsonl @@ -0,0 +1,1196 @@ +{"id": "adff4eabe2639c0c128c_en", "retrieved_id": "130f7bdeb1d62731f52f_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "adff4eabe2639c0c128c_noisy_en", "retrieved_id": "88a8a9447d5602b34319_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "adff4eabe2639c0c128c_hi", "retrieved_id": "02501bf0d6866d9d8bf4_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "adff4eabe2639c0c128c_hinglish", "retrieved_id": "484e67e978b01025c01e_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "efee65268e6dc2bb069e_en", "retrieved_id": "427c52f3cda588a8dfa5_concrete_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "efee65268e6dc2bb069e_noisy_en", "retrieved_id": "18e398d9027cc6e6bfb3_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT customer_name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "efee65268e6dc2bb069e_hi", "retrieved_id": "3aac2f1448f136869091_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT customer_name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "efee65268e6dc2bb069e_hinglish", "retrieved_id": "48a45d08da4a0866d65c_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT customer_name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f9fa9654c0e96fd0477b_en", "retrieved_id": "464d252657fc1d8b618b_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "f9fa9654c0e96fd0477b_noisy_en", "retrieved_id": "4873340f3a83e0e79dde_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM inspections WHERE region = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "f9fa9654c0e96fd0477b_hi", "retrieved_id": "a4788801036418465a47_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "f9fa9654c0e96fd0477b_hinglish", "retrieved_id": "c0ac00b959079db710e2_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE region = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2c31add7d7b45843dec1_en", "retrieved_id": "53759e65346adaac1c7b_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM inspections WHERE kind = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2c31add7d7b45843dec1_noisy_en", "retrieved_id": "6f49986bb93bef798e08_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM inspections WHERE kind = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2c31add7d7b45843dec1_hi", "retrieved_id": "6dd4bebcabfe71c95442_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM inspections WHERE kind = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2c31add7d7b45843dec1_hinglish", "retrieved_id": "53759e65346adaac1c7b_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM inspections WHERE kind = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "82c104614df1ce6e2762_en", "retrieved_id": "bd33ce4258255d7b00e8_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE salary <= 100;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "82c104614df1ce6e2762_noisy_en", "retrieved_id": "c01c73c53af1758355fb_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE category = 'South' AND salary > 100;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "82c104614df1ce6e2762_hi", "retrieved_id": "93630964314d2a1070ff_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE name IS NULL;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "82c104614df1ce6e2762_hinglish", "retrieved_id": "bf4e2d905e23df3524d7_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE salary < 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "35406fa024e66bcb5dd7_en", "retrieved_id": "901f5c9bdf09c957e942_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE score < 10000;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "35406fa024e66bcb5dd7_noisy_en", "retrieved_id": "8b652966d1e875c9cd45_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE score < 10000;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "35406fa024e66bcb5dd7_hi", "retrieved_id": "1c24e12b594ff106ec5c_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE score < 10000;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "35406fa024e66bcb5dd7_hinglish", "retrieved_id": "0264b186c6654efa83ac_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE score < 10000;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0b8db53d711baed4f4ed_en", "retrieved_id": "ee309d02322a1ea31a21_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE cost > 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "0b8db53d711baed4f4ed_noisy_en", "retrieved_id": "5c402e1eb92eb1a39327_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE cost >= 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0b8db53d711baed4f4ed_hi", "retrieved_id": "2aed937becc82c3dd876_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE cost < 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "0b8db53d711baed4f4ed_hinglish", "retrieved_id": "3c390ab6e128eb92a927_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE cost >= 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b02f421828127e2eb96b_en", "retrieved_id": "dff90759b333fcdd6daf_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "b02f421828127e2eb96b_noisy_en", "retrieved_id": "7db0d30998f0e2784d1e_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price <= 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b02f421828127e2eb96b_hi", "retrieved_id": "4c32cae59bfd87d63c78_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price >= 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "b02f421828127e2eb96b_hinglish", "retrieved_id": "51153aecdccc6fa04a6e_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price <= 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c01aaa64706cccb8ac54_en", "retrieved_id": "ce663569111e9035dd99_concrete_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE balance BETWEEN 50 AND 60;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "c01aaa64706cccb8ac54_noisy_en", "retrieved_id": "3701b333a4f0f30c5188_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE balance BETWEEN 0 AND 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c01aaa64706cccb8ac54_hi", "retrieved_id": "80a601332d98498fd1a8_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE balance BETWEEN 50 AND 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "c01aaa64706cccb8ac54_hinglish", "retrieved_id": "be113208b55b74745fca_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE balance BETWEEN 0 AND 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0b8689cf8e381cbcfe86_en", "retrieved_id": "ddff6003a3c7266f9a3a_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE department = 'pending' AND price > 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0b8689cf8e381cbcfe86_noisy_en", "retrieved_id": "415f5c285edc8c2b092c_en_curriculum_99606", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE department = 'pending' AND price > 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0b8689cf8e381cbcfe86_hi", "retrieved_id": "ddff6003a3c7266f9a3a_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE department = 'pending' AND price > 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0b8689cf8e381cbcfe86_hinglish", "retrieved_id": "8f717cd8f43ba6663d64_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE department = 'pending' AND price > 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6f34909f5b1104bf7ac1_en", "retrieved_id": "7151a21566102fe10631_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE region IN ('pending', 'pending');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "6f34909f5b1104bf7ac1_noisy_en", "retrieved_id": "7151a21566102fe10631_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE region IN ('pending', 'pending');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "6f34909f5b1104bf7ac1_hi", "retrieved_id": "943cc39b7fa48e12a8cf_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE region IN ('Delhi', 'pending');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6f34909f5b1104bf7ac1_hinglish", "retrieved_id": "469efb097cab19d7331b_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE region IN ('Delhi', 'pending');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2e2ac3c5dcf95429daaa_en", "retrieved_id": "bda2d640bb8440ba316d_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2e2ac3c5dcf95429daaa_noisy_en", "retrieved_id": "2979500f0ecfcf0ceeaf_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2e2ac3c5dcf95429daaa_hi", "retrieved_id": "6f58fb9ba5d390237de3_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE NOT title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "2e2ac3c5dcf95429daaa_hinglish", "retrieved_id": "5bb801582af241e0e0d2_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE NOT title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "e7d754ed36b247db55d3_en", "retrieved_id": "993d6dbb886fdcff8483_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE NOT description IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "e7d754ed36b247db55d3_noisy_en", "retrieved_id": "52c6084dc05c08ecedad_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE description IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "e7d754ed36b247db55d3_hi", "retrieved_id": "36846cba4b79e7ab511e_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE description IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e7d754ed36b247db55d3_hinglish", "retrieved_id": "006e0510328e4f2c7c4a_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE description IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "40464b00088c9159e3d0_en", "retrieved_id": "c40602913c502e7ccda9_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT COUNT(*) FROM inspections WHERE department = 'Mumbai';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "40464b00088c9159e3d0_noisy_en", "retrieved_id": "6040c6883947ec5b3821_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM inspections;\",\"project_id\":\"demo_inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "40464b00088c9159e3d0_hi", "retrieved_id": "2d416c705002edc7a1af_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM inspections;\",\"project_id\":\"demo_inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "40464b00088c9159e3d0_hinglish", "retrieved_id": "33e0077bee491c810f9d_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM inspections;\",\"project_id\":\"demo_inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "80ed46f6e275ed01c6c1_en", "retrieved_id": "f2470e286894388c7f5c_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM exhibits WHERE category = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "80ed46f6e275ed01c6c1_noisy_en", "retrieved_id": "d579114cb797a057c40a_concrete_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM exhibits WHERE category = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "80ed46f6e275ed01c6c1_hi", "retrieved_id": "3fbc5d2132a64d443cd1_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE category IN ('Pune', 'Mumbai');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "80ed46f6e275ed01c6c1_hinglish", "retrieved_id": "6efaf4bfcca4eef4eb82_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM exhibits WHERE category = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d735c1ac1979ff1e8620_en", "retrieved_id": "e4a4498900568bd25b0f_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT kind, SUM(price) FROM inspections GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "d735c1ac1979ff1e8620_noisy_en", "retrieved_id": "a33bc62f70795ecfa8a2_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(price) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d735c1ac1979ff1e8620_hi", "retrieved_id": "3c25f8671f95114b6257_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(price) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d735c1ac1979ff1e8620_hinglish", "retrieved_id": "caba0067fdd24c562152_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(price) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8efcf517aee64807a8e5_en", "retrieved_id": "a1369f19e8dcd07436c9_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8efcf517aee64807a8e5_noisy_en", "retrieved_id": "e16c4dae0f3afac7f310_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8efcf517aee64807a8e5_hi", "retrieved_id": "52abab06839657236449_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8efcf517aee64807a8e5_hinglish", "retrieved_id": "3442026d043514bd5f1c_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "cd5bb3382b9e6c8604e8_en", "retrieved_id": "f799074a7799eb92b1c0_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(rating) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "cd5bb3382b9e6c8604e8_noisy_en", "retrieved_id": "ad5c6763cee83fde6df0_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(rating) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "cd5bb3382b9e6c8604e8_hi", "retrieved_id": "2bd4dee285ff24521ff6_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(rating) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "cd5bb3382b9e6c8604e8_hinglish", "retrieved_id": "29e87bc0175bf44f2332_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(rating) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7094d70d08791284b632_en", "retrieved_id": "5e2e49adf71668e6010b_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(salary) FROM reservations;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7094d70d08791284b632_noisy_en", "retrieved_id": "6c77dd495d7091960ccf_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(salary) FROM reservations;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7094d70d08791284b632_hi", "retrieved_id": "57560f761f427dc6f2ad_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(salary) FROM reservations;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7094d70d08791284b632_hinglish", "retrieved_id": "9e2da52aa50149315527_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(salary) FROM reservations;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "cab55a7429427fbc5093_en", "retrieved_id": "f4d0cc8f44c31aba25fd_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "cab55a7429427fbc5093_noisy_en", "retrieved_id": "f4d0cc8f44c31aba25fd_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "cab55a7429427fbc5093_hi", "retrieved_id": "f4d0cc8f44c31aba25fd_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "cab55a7429427fbc5093_hinglish", "retrieved_id": "f0e8a1aa17ebe2943f9a_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE customer_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "28a7ce61b1233483bf73_en", "retrieved_id": "de723244833eb9acd9b0_concrete_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "28a7ce61b1233483bf73_noisy_en", "retrieved_id": "dbbc87df17789dcf16c7_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY salary DESC NULLS LAST;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "28a7ce61b1233483bf73_hi", "retrieved_id": "5f714fc20f63b02a2389_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY salary DESC NULLS LAST;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "28a7ce61b1233483bf73_hinglish", "retrieved_id": "9360abd24d181d80b919_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY salary DESC NULLS LAST;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "804d318fe46a0e664840_en", "retrieved_id": "9893daa34daf0837aacd_concrete_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY CASE WHEN quantity IS NULL THEN 1 ELSE 0 END DESC, quantity DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "804d318fe46a0e664840_noisy_en", "retrieved_id": "08cdefb37331da7aacab_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY quantity DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "804d318fe46a0e664840_hi", "retrieved_id": "e31c637236043ab2d50e_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY quantity ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "804d318fe46a0e664840_hinglish", "retrieved_id": "bd6dcbc8d4b468bfdc18_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY quantity ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b9c176c07809f42564a0_en", "retrieved_id": "972c5811c083df6a38a4_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY CASE WHEN amount IS NULL THEN 1 ELSE 0 END DESC, amount DESC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "b9c176c07809f42564a0_noisy_en", "retrieved_id": "a660459963aa550973d6_concrete_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "b9c176c07809f42564a0_hi", "retrieved_id": "308f3acec7c873c00019_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b9c176c07809f42564a0_hinglish", "retrieved_id": "72d5e4666d38219ea72f_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1a1af4deb475f0224a59_en", "retrieved_id": "e6483bc77519e5b72fc9_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY price ASC LIMIT 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1a1af4deb475f0224a59_noisy_en", "retrieved_id": "e6483bc77519e5b72fc9_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY price ASC LIMIT 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1a1af4deb475f0224a59_hi", "retrieved_id": "fdc1f5ca0b128c6bbb57_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY price ASC NULLS FIRST LIMIT 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "1a1af4deb475f0224a59_hinglish", "retrieved_id": "8ff20ffc24a07df327c1_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY price ASC LIMIT 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "475287613764159ee5eb_en", "retrieved_id": "39ffd26f283fc2242233_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "475287613764159ee5eb_noisy_en", "retrieved_id": "6ad175c1da38937a1c73_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "475287613764159ee5eb_hi", "retrieved_id": "11be592bce7c6de29c95_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "475287613764159ee5eb_hinglish", "retrieved_id": "c2e6ba167b989ceb36a1_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6972a0b38fd4a8758723_en", "retrieved_id": "0e35aa356b3a7da6b2bd_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT kind, SUM(score) FROM exhibits GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6972a0b38fd4a8758723_noisy_en", "retrieved_id": "9917d9d119837017ff3c_concrete_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT kind, SUM(score) FROM exhibits GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6972a0b38fd4a8758723_hi", "retrieved_id": "090cc722d12e08b58b91_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT kind, SUM(score) FROM exhibits GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6972a0b38fd4a8758723_hinglish", "retrieved_id": "92e9f6f1139932d803a6_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT kind, SUM(score) FROM exhibits GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8a2e6d77aa1c3a517ecb_en", "retrieved_id": "de4dcd8a722b64c83c8e_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM exhibits GROUP BY category HAVING COUNT(*) > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8a2e6d77aa1c3a517ecb_noisy_en", "retrieved_id": "de4dcd8a722b64c83c8e_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM exhibits GROUP BY category HAVING COUNT(*) > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8a2e6d77aa1c3a517ecb_hi", "retrieved_id": "3b11560f58cefc649e43_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM exhibits GROUP BY category HAVING COUNT(*) > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8a2e6d77aa1c3a517ecb_hinglish", "retrieved_id": "10b9b5ec1ce9a4d6efa8_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM exhibits GROUP BY category HAVING COUNT(*) > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1d82ef2b9527e797c466_en", "retrieved_id": "15a9906bc7837861cefd_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE order_date >= '2022-04-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1d82ef2b9527e797c466_noisy_en", "retrieved_id": "15a9906bc7837861cefd_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE order_date >= '2022-04-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1d82ef2b9527e797c466_hi", "retrieved_id": "ad81c76755fa18bd34ad_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE order_date >= '2022-04-15';\",\"project_id\":\"demo_inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1d82ef2b9527e797c466_hinglish", "retrieved_id": "7625e04b50251ffe051b_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE order_date >= '2022-04-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d8f9a887c52e766a8e14_en", "retrieved_id": "000a80ab7250068ca30c_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE EXTRACT(YEAR FROM start_date) = 2023;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "d8f9a887c52e766a8e14_noisy_en", "retrieved_id": "09f4415c5bcb0fa139a4_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE YEAR(start_date) = 2023;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d8f9a887c52e766a8e14_hi", "retrieved_id": "66bbdcc97ef1052504f5_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE EXTRACT(YEAR FROM start_date) = 2023;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "d8f9a887c52e766a8e14_hinglish", "retrieved_id": "944fbd8ddfee60f00abd_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE EXTRACT(YEAR FROM start_date) = 2023;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "f35f7a78a14e903920ee_en", "retrieved_id": "6d8b449cc509987480e0_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE EXTRACT(MONTH FROM order_date) = 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f35f7a78a14e903920ee_noisy_en", "retrieved_id": "61d7e4ed449ac17088cd_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE EXTRACT(MONTH FROM order_date) = 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f35f7a78a14e903920ee_hi", "retrieved_id": "af68663d985ebce594ef_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE EXTRACT(MONTH FROM CAST(order_date AS DATE)) = 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "f35f7a78a14e903920ee_hinglish", "retrieved_id": "7636319c3fba9456741e_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE EXTRACT(MONTH FROM CAST(order_date AS DATE)) = 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "26db633822b8319f83ec_en", "retrieved_id": "de493284918940b1fdf1_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE LOWER(name) LIKE LOWER('%active%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "26db633822b8319f83ec_noisy_en", "retrieved_id": "8557de97b7434c6c3e04_concrete_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE LOWER(name) LIKE LOWER('%active%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "26db633822b8319f83ec_hi", "retrieved_id": "675e1f645410b7ed7cf4_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE LOWER(name) LIKE LOWER('%active%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "26db633822b8319f83ec_hinglish", "retrieved_id": "d1ecd6cc31c228d0a25b_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE LOWER(name) LIKE LOWER('%active%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d1f372f544fa10f6df1b_en", "retrieved_id": "6b294e1fcd8c647e8781_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT reservations.title, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d1f372f544fa10f6df1b_noisy_en", "retrieved_id": "6b294e1fcd8c647e8781_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT reservations.title, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d1f372f544fa10f6df1b_hi", "retrieved_id": "8509bb2979fbeb8ee4c7_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT reservations.title, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d1f372f544fa10f6df1b_hinglish", "retrieved_id": "d22704e7765e8c67ae4b_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT reservations.title, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a30eeaf286c79c95a1cd_en", "retrieved_id": "9e1f3d2de009fd685c86_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT inspections.item_name FROM inspections JOIN inspections_groups ON inspections.group_id = inspections_groups.id WHERE inspections_groups.label = 'O''Reilly';\",\"project_id\":\"demo_inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a30eeaf286c79c95a1cd_noisy_en", "retrieved_id": "6dff72b0eece1766b6f4_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT inspections.item_name FROM inspections JOIN inspections_groups ON inspections.group_id = inspections_groups.id WHERE inspections_groups.label = 'O''Reilly';\",\"project_id\":\"demo_inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a30eeaf286c79c95a1cd_hi", "retrieved_id": "61ae49349949dc399a63_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT inspections.item_name FROM inspections JOIN inspections_groups ON inspections.group_id = inspections_groups.id WHERE inspections_groups.label = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a30eeaf286c79c95a1cd_hinglish", "retrieved_id": "e60aa1d2eef307b6933f_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT inspections.item_name FROM inspections JOIN inspections_groups ON inspections.group_id = inspections_groups.id WHERE inspections_groups.label = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5a8f2e207d0efe5b018c_en", "retrieved_id": "bd6b78905a3ad03e04b6_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specimens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "5a8f2e207d0efe5b018c_noisy_en", "retrieved_id": "bd6b78905a3ad03e04b6_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specimens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "5a8f2e207d0efe5b018c_hi", "retrieved_id": "944772e48c9b5194fe1e_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specimens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "5a8f2e207d0efe5b018c_hinglish", "retrieved_id": "5165d04cc5f4626621a5_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"],\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "e62cfd2a6789616a503f_en", "retrieved_id": "863d3ef71bbbeba4919e_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "e62cfd2a6789616a503f_noisy_en", "retrieved_id": "632bfa2e24875823150a_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "e62cfd2a6789616a503f_hi", "retrieved_id": "286869979f46e9bbc61c_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "e62cfd2a6789616a503f_hinglish", "retrieved_id": "4253da29b10c8d2d921b_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"books\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "84b16996198346dc9882_en", "retrieved_id": "fb890425dce9de189858_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "84b16996198346dc9882_noisy_en", "retrieved_id": "25ac9041dfe1446dd870_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "84b16996198346dc9882_hi", "retrieved_id": "427fd58d6660abf6f38a_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "84b16996198346dc9882_hinglish", "retrieved_id": "25ac9041dfe1446dd870_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "30dd94e63a065d87109e_en", "retrieved_id": "ff73314a0982d6281641_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"books\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "30dd94e63a065d87109e_noisy_en", "retrieved_id": "03727992a863b145c565_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "30dd94e63a065d87109e_hi", "retrieved_id": "26ee7e71ccaee33a6162_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "30dd94e63a065d87109e_hinglish", "retrieved_id": "0af23c07619fdf39d636_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"books\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "c4210296305105ab7472_en", "retrieved_id": "a5928e9d2a9de389c96e_noisy_en", "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "c4210296305105ab7472_noisy_en", "retrieved_id": "7b86419573eea700d641_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": false, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false}} +{"id": "c4210296305105ab7472_hi", "retrieved_id": "d8fbf98bcb77ca92a390_hi", "output": "{\"action\":\"clarify\",\"question\":\"सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "c4210296305105ab7472_hinglish", "retrieved_id": "981c50947db171d5fec5_hinglish", "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "3fdbd39c598af9b9ea10_en", "retrieved_id": "b477a5e30a03db87c812_noisy_en", "output": "{\"action\":\"clarify\",\"question\":\"Which value should status equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "3fdbd39c598af9b9ea10_noisy_en", "retrieved_id": "b477a5e30a03db87c812_noisy_en", "output": "{\"action\":\"clarify\",\"question\":\"Which value should status equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "3fdbd39c598af9b9ea10_hi", "retrieved_id": "0728bb1421323cd56a8c_hi", "output": "{\"action\":\"clarify\",\"question\":\"status का कौन सा मान चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "3fdbd39c598af9b9ea10_hinglish", "retrieved_id": "511647de271f97585893_hinglish", "output": "{\"action\":\"clarify\",\"question\":\"status ki kaunsi value chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "2162ee1266aed9320937_en", "retrieved_id": "8fde1669f8ae856382b5_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": false, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false}} +{"id": "2162ee1266aed9320937_noisy_en", "retrieved_id": "f6d1979b8e8273fb994b_en", "output": "{\"action\":\"clarify\",\"question\":\"This connection is read-only. Would you like a SELECT query instead?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "2162ee1266aed9320937_hi", "retrieved_id": "a4f18f2163d8129dfc3c_hi", "output": "{\"action\":\"clarify\",\"question\":\"यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "2162ee1266aed9320937_hinglish", "retrieved_id": "6cad228416bb08b460f6_hinglish", "output": "{\"action\":\"clarify\",\"question\":\"Yeh connection read-only hai. Kya SELECT query chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "754691bd1eea25d41ccf_en", "retrieved_id": "1173d51478c0f0770b25_en", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "754691bd1eea25d41ccf_noisy_en", "retrieved_id": "8f78f2ccd606ac642613_noisy_en", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "754691bd1eea25d41ccf_hi", "retrieved_id": "04fd2c1a67c04de23381_hi", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "754691bd1eea25d41ccf_hinglish", "retrieved_id": "240cb7bac57363d0776e_hinglish", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "252c6053295d69a93c94_en", "retrieved_id": "67e8af086aba543f5377_en", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "252c6053295d69a93c94_noisy_en", "retrieved_id": "6e9991cb6cf8acb8aa01_en", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "252c6053295d69a93c94_hi", "retrieved_id": "e1d55e63211c7100096f_hi", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "252c6053295d69a93c94_hinglish", "retrieved_id": "f9004fd3e12dbc0d88da_hinglish", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "f73e21e0eb10d6617995_en", "retrieved_id": "3662e94e29edbee93120_en", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "f73e21e0eb10d6617995_noisy_en", "retrieved_id": "3662e94e29edbee93120_en", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "f73e21e0eb10d6617995_hi", "retrieved_id": "28f97a3353e78f589cfe_hi", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "f73e21e0eb10d6617995_hinglish", "retrieved_id": "0e78e9ded20fe5272477_hinglish", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "8a696e3c3522dd69d5ac_en", "retrieved_id": "7433354d41a6c516f4b6_en", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-8757\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "8a696e3c3522dd69d5ac_noisy_en", "retrieved_id": "4c422d229cf597a3acdd_noisy_en_curriculum_99606", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-117\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "8a696e3c3522dd69d5ac_hi", "retrieved_id": "b0c31fd3cb1afd956208_hi", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-9402\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "8a696e3c3522dd69d5ac_hinglish", "retrieved_id": "6ea26747bfbf3b1aa5e0_hinglish", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-8757\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "797b9b914530f2eb502a_en", "retrieved_id": "8cb26e68080051ad3619_noisy_en", "output": "{\"action\":\"answer\",\"text\":\"The tool returned 48 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "797b9b914530f2eb502a_noisy_en", "retrieved_id": "8cb26e68080051ad3619_noisy_en", "output": "{\"action\":\"answer\",\"text\":\"The tool returned 48 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "797b9b914530f2eb502a_hi", "retrieved_id": "630408eb6f8f54f3bc28_hi", "output": "{\"action\":\"answer\",\"text\":\"टूल ने 48 पंक्तियाँ लौटाईं।\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "797b9b914530f2eb502a_hinglish", "retrieved_id": "e029eeba4f40f3109a40_hinglish", "output": "{\"action\":\"answer\",\"text\":\"Tool ne 2 rows return ki.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false, "text_exact": false}} +{"id": "2ad3e251c936135e2005_en", "retrieved_id": "130f7bdeb1d62731f52f_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2ad3e251c936135e2005_noisy_en", "retrieved_id": "88a8a9447d5602b34319_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2ad3e251c936135e2005_hi", "retrieved_id": "02501bf0d6866d9d8bf4_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2ad3e251c936135e2005_hinglish", "retrieved_id": "484e67e978b01025c01e_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a060620c638d2dc3759e_en", "retrieved_id": "427c52f3cda588a8dfa5_concrete_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT DISTINCT region FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "a060620c638d2dc3759e_noisy_en", "retrieved_id": "f158884fc953e9bd6dc7_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a060620c638d2dc3759e_hi", "retrieved_id": "3aac2f1448f136869091_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a060620c638d2dc3759e_hinglish", "retrieved_id": "1e5a4a69bda180f02aa2_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c8fd1555fd64b08f1241_en", "retrieved_id": "578556e0f76a1becdc8b_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM inspections WHERE city = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "c8fd1555fd64b08f1241_noisy_en", "retrieved_id": "6be1b6edf9b1b306a785_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM inspections WHERE city = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "c8fd1555fd64b08f1241_hi", "retrieved_id": "ff935e3d50ebaf0b8c53_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM inspections WHERE city = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "c8fd1555fd64b08f1241_hinglish", "retrieved_id": "16926bb9c79e82b85097_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE city = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "eb982cf071094adaca4f_en", "retrieved_id": "f0fd5b687a47200d08cc_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT name FROM reservations WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "eb982cf071094adaca4f_noisy_en", "retrieved_id": "f0fd5b687a47200d08cc_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT name FROM reservations WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "eb982cf071094adaca4f_hi", "retrieved_id": "c010cb1b1703bc1b48c3_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT name FROM reservations WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "eb982cf071094adaca4f_hinglish", "retrieved_id": "dc119519e99b6db7717d_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT name FROM reservations WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c023a5b6f95173ae856d_en", "retrieved_id": "725bc4721dea275762a0_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE score <= 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "c023a5b6f95173ae856d_noisy_en", "retrieved_id": "b29589b85da26af925a2_concrete_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE score > 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "c023a5b6f95173ae856d_hi", "retrieved_id": "93630964314d2a1070ff_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE display_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "c023a5b6f95173ae856d_hinglish", "retrieved_id": "b2684d967d976714b34c_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE score < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "affa5e5e14444cd82bb4_en", "retrieved_id": "6a3bbef7eab5a98a8204_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE salary < 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "affa5e5e14444cd82bb4_noisy_en", "retrieved_id": "6adee58250bc9b3b92ca_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE salary < 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "affa5e5e14444cd82bb4_hi", "retrieved_id": "6a3bbef7eab5a98a8204_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE salary < 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "affa5e5e14444cd82bb4_hinglish", "retrieved_id": "6a3bbef7eab5a98a8204_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE salary < 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7d4234476674a04380e6_en", "retrieved_id": "d898569e45422c9376f7_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE score > 1500;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "7d4234476674a04380e6_noisy_en", "retrieved_id": "18b58f4b6324bf5e24ef_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE score < 1500;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "7d4234476674a04380e6_hi", "retrieved_id": "34c472fb5ac8389fdcd4_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE score < 1500;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "7d4234476674a04380e6_hinglish", "retrieved_id": "9a18682fc49a50e8ab0f_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE score >= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c4f25575abfd24e3cd63_en", "retrieved_id": "94b9e21d18d901c193f0_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE cost > 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "c4f25575abfd24e3cd63_noisy_en", "retrieved_id": "8c2862d050c86d717184_concrete_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE cost <= 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "c4f25575abfd24e3cd63_hi", "retrieved_id": "55443a8bb63ef75dfd2c_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE cost >= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "c4f25575abfd24e3cd63_hinglish", "retrieved_id": "44b7c6f0585479334d72_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE cost <= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0d94a075cd7e5759bdfa_en", "retrieved_id": "93abf834af493d9e949e_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE rating BETWEEN 10 AND 110;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0d94a075cd7e5759bdfa_noisy_en", "retrieved_id": "c84607652e4c589161a1_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE rating BETWEEN 10 AND 110;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0d94a075cd7e5759bdfa_hi", "retrieved_id": "c84607652e4c589161a1_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE rating BETWEEN 10 AND 110;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0d94a075cd7e5759bdfa_hinglish", "retrieved_id": "93abf834af493d9e949e_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE rating BETWEEN 10 AND 110;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1695d78b68e4d8fca6f0_en", "retrieved_id": "29412c1b0f2e0c5d1775_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE region = 'Books' AND amount > 1500;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "1695d78b68e4d8fca6f0_noisy_en", "retrieved_id": "9a27894b4516f46b13a1_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE region = 'Delhi' AND amount > 50;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "1695d78b68e4d8fca6f0_hi", "retrieved_id": "b124ea61e45d5e464010_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE region = 'Delhi' AND amount > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1695d78b68e4d8fca6f0_hinglish", "retrieved_id": "b124ea61e45d5e464010_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE region = 'Delhi' AND amount > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "45c3d3db5419d8a8dfeb_en", "retrieved_id": "d45c4afdf3b0ab57857f_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE kind IN ('O''Reilly', 'Mumbai');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "45c3d3db5419d8a8dfeb_noisy_en", "retrieved_id": "d45c4afdf3b0ab57857f_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE kind IN ('O''Reilly', 'Mumbai');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "45c3d3db5419d8a8dfeb_hi", "retrieved_id": "d45c4afdf3b0ab57857f_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE kind IN ('O''Reilly', 'Mumbai');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "45c3d3db5419d8a8dfeb_hinglish", "retrieved_id": "d45c4afdf3b0ab57857f_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE kind IN ('O''Reilly', 'Mumbai');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "8b1939fffee21884f667_en", "retrieved_id": "b938809a21ca895bfdaa_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE item_name IS NULL;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8b1939fffee21884f667_noisy_en", "retrieved_id": "999181602f630fe41f53_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8b1939fffee21884f667_hi", "retrieved_id": "f1b3784f5142d5df00b8_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE item_name IS NOT NULL;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "8b1939fffee21884f667_hinglish", "retrieved_id": "c39505a9b595f6ac6b82_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE item_name IS NULL;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c50090341e5d5516a579_en", "retrieved_id": "0e4b6bf620f2b93f8329_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE customer_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c50090341e5d5516a579_noisy_en", "retrieved_id": "a50de851e3ed623c862c_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE customer_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "c50090341e5d5516a579_hi", "retrieved_id": "4c315741ef5c61a725f7_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE NOT customer_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "c50090341e5d5516a579_hinglish", "retrieved_id": "deed3e34bcd12abea394_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE NOT customer_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "23c0aa3a4596f995ad2a_en", "retrieved_id": "c40602913c502e7ccda9_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens WHERE category = 'Mumbai';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "23c0aa3a4596f995ad2a_noisy_en", "retrieved_id": "6040c6883947ec5b3821_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "23c0aa3a4596f995ad2a_hi", "retrieved_id": "2d416c705002edc7a1af_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "23c0aa3a4596f995ad2a_hinglish", "retrieved_id": "33e0077bee491c810f9d_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "81b12886ddb859bb71db_en", "retrieved_id": "cbc3795c0ca21a9b8c5a_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "81b12886ddb859bb71db_noisy_en", "retrieved_id": "8463566151edd133626c_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM inspections WHERE city = 'pending';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "81b12886ddb859bb71db_hi", "retrieved_id": "60a34656f2f657c68d8a_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE city IN ('pending', 'pending');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "81b12886ddb859bb71db_hinglish", "retrieved_id": "8f46a1742f10c08d6c17_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM inspections WHERE city = 'pending';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1896aafc31bd0a7d24ec_en", "retrieved_id": "9a941e036270f910961f_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT region, SUM(balance) FROM exhibits GROUP BY region;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "1896aafc31bd0a7d24ec_noisy_en", "retrieved_id": "f33cbbe711d8d7aac49c_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(balance) FROM exhibits;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1896aafc31bd0a7d24ec_hi", "retrieved_id": "e96bc561fa378ade1824_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(balance) FROM exhibits;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1896aafc31bd0a7d24ec_hinglish", "retrieved_id": "4c1c4532a70790cdae6c_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT SUM(balance) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b564e1a7b1e7fc6d2ab4_en", "retrieved_id": "5c7028d7d36fd491bc2e_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT AVG(score) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b564e1a7b1e7fc6d2ab4_noisy_en", "retrieved_id": "a441a9c205497a51a291_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(score) FROM specimens;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b564e1a7b1e7fc6d2ab4_hi", "retrieved_id": "5c7028d7d36fd491bc2e_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT AVG(score) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b564e1a7b1e7fc6d2ab4_hinglish", "retrieved_id": "6bb8ee776b961884c402_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(score) FROM specimens;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7506912696ddf94cfae4_en", "retrieved_id": "89700fa6c178672acbb9_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(amount) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7506912696ddf94cfae4_noisy_en", "retrieved_id": "7e451df33141f550eb0b_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(amount) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7506912696ddf94cfae4_hi", "retrieved_id": "73cca5dd8b7549a2ade7_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(amount) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "7506912696ddf94cfae4_hinglish", "retrieved_id": "0478666ddfaf5a23162e_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(amount) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "9aed1992a793fbf97601_en", "retrieved_id": "d5cc3d76002f556ee9d3_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT MIN(price) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9aed1992a793fbf97601_noisy_en", "retrieved_id": "da9e0a0592e6238536af_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT MIN(price) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9aed1992a793fbf97601_hi", "retrieved_id": "e0e3d410c4591e668ca3_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT MIN(price) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9aed1992a793fbf97601_hinglish", "retrieved_id": "bdf01655d06fc4dc9de0_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT MIN(price) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8a3194b9f492b50b8100_en", "retrieved_id": "c6325f377e0c7f0468f1_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT category FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8a3194b9f492b50b8100_noisy_en", "retrieved_id": "c6325f377e0c7f0468f1_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT category FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8a3194b9f492b50b8100_hi", "retrieved_id": "ce1d19d1f1e239c03e1b_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT category FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8a3194b9f492b50b8100_hinglish", "retrieved_id": "f0e8a1aa17ebe2943f9a_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "e92015bce168f8f9a90a_en", "retrieved_id": "eae5287706bec5ee520e_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY score DESC;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e92015bce168f8f9a90a_noisy_en", "retrieved_id": "955a7d5f0d3cb038c3fa_concrete_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY score ASC NULLS FIRST;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "e92015bce168f8f9a90a_hi", "retrieved_id": "190107647feb114e3236_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY score DESC NULLS LAST;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "e92015bce168f8f9a90a_hinglish", "retrieved_id": "9360abd24d181d80b919_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY score DESC NULLS LAST;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "b30330bb2c17b32c6883_en", "retrieved_id": "56cddde09b6ce33e29af_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY cost DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "b30330bb2c17b32c6883_noisy_en", "retrieved_id": "fd6c642df54b72b5146f_concrete_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY cost ASC NULLS FIRST;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "b30330bb2c17b32c6883_hi", "retrieved_id": "3de06ebc9c472cfcc5f5_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY cost ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b30330bb2c17b32c6883_hinglish", "retrieved_id": "0f074b2420cf3783035e_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY cost ASC NULLS FIRST;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "5890e52f983d16adbae7_en", "retrieved_id": "955a7d5f0d3cb038c3fa_concrete_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY quantity ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "5890e52f983d16adbae7_noisy_en", "retrieved_id": "1b2780774c2ef3389909_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY CASE WHEN quantity IS NULL THEN 1 ELSE 0 END DESC, quantity DESC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "5890e52f983d16adbae7_hi", "retrieved_id": "36a2047a9b8d5e928dc2_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY quantity DESC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5890e52f983d16adbae7_hinglish", "retrieved_id": "2d0bbdceb628c88c79ac_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY quantity DESC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bd760f1ca7aff546e852_en", "retrieved_id": "bd8d48d0d5b50e6d28e3_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits ORDER BY balance ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bd760f1ca7aff546e852_noisy_en", "retrieved_id": "bd8d48d0d5b50e6d28e3_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits ORDER BY balance ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bd760f1ca7aff546e852_hi", "retrieved_id": "f6733b4a160882f7ede6_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits ORDER BY balance ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bd760f1ca7aff546e852_hinglish", "retrieved_id": "bd8d48d0d5b50e6d28e3_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits ORDER BY balance ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2164b376f732a1c371ab_en", "retrieved_id": "0517beaf82464fadc60f_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM specimens GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2164b376f732a1c371ab_noisy_en", "retrieved_id": "70063552c331bd678a4c_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM specimens GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2164b376f732a1c371ab_hi", "retrieved_id": "00b4a7ffa4e87f1134ce_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM specimens GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2164b376f732a1c371ab_hinglish", "retrieved_id": "70f425096c4a3fdd8159_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM specimens GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5d97abdb52189efbe117_en", "retrieved_id": "f03903b51d3bff7de034_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT status, SUM(amount) FROM exhibits GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5d97abdb52189efbe117_noisy_en", "retrieved_id": "8db52bac8a3a32a185cc_concrete_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT status, SUM(amount) FROM exhibits GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5d97abdb52189efbe117_hi", "retrieved_id": "82ed28cbc1367e475caf_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT status, SUM(amount) FROM exhibits GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5d97abdb52189efbe117_hinglish", "retrieved_id": "ba583b46c31284cbf512_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT status, SUM(amount) FROM exhibits GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "01f4f9527d44f09ab6b6_en", "retrieved_id": "6c61d9e591b587769eba_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM reservations GROUP BY kind HAVING COUNT(*) > 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "01f4f9527d44f09ab6b6_noisy_en", "retrieved_id": "64ef705c22e6cf4b7a09_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM reservations GROUP BY kind HAVING COUNT(*) > 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "01f4f9527d44f09ab6b6_hi", "retrieved_id": "c055c000835e9f443bd4_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM reservations GROUP BY kind HAVING COUNT(*) > 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "01f4f9527d44f09ab6b6_hinglish", "retrieved_id": "72bb694e41e56f9df740_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM reservations GROUP BY kind HAVING COUNT(*) > 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "58eb6ab2fb9c69becdbd_en", "retrieved_id": "f6ae19804fd6ac3ced82_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE created_at >= '2022-08-15';\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "58eb6ab2fb9c69becdbd_noisy_en", "retrieved_id": "21275fc52796db99c79d_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE created_at >= '2022-08-15';\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "58eb6ab2fb9c69becdbd_hi", "retrieved_id": "1b1174ed243b6358705a_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE created_at >= '2026-08-15';\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "58eb6ab2fb9c69becdbd_hinglish", "retrieved_id": "42c7c34ccdbb091be000_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE created_at >= '2022-02-15';\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "ed474bd8a459cbfaa5d2_en", "retrieved_id": "b4ae57998c341af42244_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE EXTRACT(YEAR FROM CAST(recorded_at AS DATE)) = 2021;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "ed474bd8a459cbfaa5d2_noisy_en", "retrieved_id": "9f9c0a9aeabb87813004_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE EXTRACT(YEAR FROM recorded_at) = 2021;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ed474bd8a459cbfaa5d2_hi", "retrieved_id": "01cdb123039aea741782_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE EXTRACT(YEAR FROM recorded_at) = 2021;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ed474bd8a459cbfaa5d2_hinglish", "retrieved_id": "af1531839f3945723f77_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE EXTRACT(YEAR FROM CAST(recorded_at AS DATE)) = 2021;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "223765dc6e033f0ab2fd_en", "retrieved_id": "4e2d0bdc05908cc767e2_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE EXTRACT(MONTH FROM created_at) = 9;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "223765dc6e033f0ab2fd_noisy_en", "retrieved_id": "c5ffbd2d231ad52e7a03_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE EXTRACT(MONTH FROM CAST(created_at AS DATE)) = 9;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "223765dc6e033f0ab2fd_hi", "retrieved_id": "030be1460de7a605e7af_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE EXTRACT(MONTH FROM CAST(created_at AS DATE)) = 9;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "223765dc6e033f0ab2fd_hinglish", "retrieved_id": "e496cf7f610533e3c3b7_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE EXTRACT(MONTH FROM created_at) = 9;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "00c24a7cc97ee9debf36_en", "retrieved_id": "cf3c59ebbb82c1fd93ff_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE LOWER(title) LIKE LOWER('%pending%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "00c24a7cc97ee9debf36_noisy_en", "retrieved_id": "279c576638ef45226927_concrete_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE LOWER(title) LIKE LOWER('%Books%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "00c24a7cc97ee9debf36_hi", "retrieved_id": "119cbd226f5ad2586c69_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE LOWER(title) LIKE LOWER('%pending%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "00c24a7cc97ee9debf36_hinglish", "retrieved_id": "781c099ea3698e1b2f6a_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE LOWER(title) LIKE LOWER('%pending%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "139a72e10281049d1fed_en", "retrieved_id": "00a9c0bb3293a7f312d2_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT specimens.name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "139a72e10281049d1fed_noisy_en", "retrieved_id": "00a9c0bb3293a7f312d2_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT specimens.name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "139a72e10281049d1fed_hi", "retrieved_id": "8509bb2979fbeb8ee4c7_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT specimens.name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "139a72e10281049d1fed_hinglish", "retrieved_id": "262a87d6e3a88b3a4d95_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT specimens.name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4a37ab647af0a5a5cb30_en", "retrieved_id": "5abcebdb0973d2af6871_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT reservations.display_name FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4a37ab647af0a5a5cb30_noisy_en", "retrieved_id": "5abcebdb0973d2af6871_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT reservations.display_name FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4a37ab647af0a5a5cb30_hi", "retrieved_id": "b8f982d31fe69c9dfe19_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT reservations.display_name FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4a37ab647af0a5a5cb30_hinglish", "retrieved_id": "72739dc137966d032522_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT reservations.display_name FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "578a0a8a98b724c87050_en", "retrieved_id": "bd6b78905a3ad03e04b6_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "578a0a8a98b724c87050_noisy_en", "retrieved_id": "bd6b78905a3ad03e04b6_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "578a0a8a98b724c87050_hi", "retrieved_id": "944772e48c9b5194fe1e_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "578a0a8a98b724c87050_hinglish", "retrieved_id": "5165d04cc5f4626621a5_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "51f14088477a5f6be9c5_en", "retrieved_id": "863d3ef71bbbeba4919e_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"],\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "51f14088477a5f6be9c5_noisy_en", "retrieved_id": "632bfa2e24875823150a_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specimens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "51f14088477a5f6be9c5_hi", "retrieved_id": "286869979f46e9bbc61c_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"],\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "51f14088477a5f6be9c5_hinglish", "retrieved_id": "4253da29b10c8d2d921b_hinglish", "output": "{\"action\":\"call\",\"name\":\"describe_table\",\"arguments\":{\"table\":\"books\"}}", "metrics": {"json_valid": true, "schema_valid": false, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false, "error": "Unknown tool"}} +{"id": "6db4ac493194a52cbd3a_en", "retrieved_id": "fb890425dce9de189858_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_reservations\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "6db4ac493194a52cbd3a_noisy_en", "retrieved_id": "25ac9041dfe1446dd870_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"],\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "6db4ac493194a52cbd3a_hi", "retrieved_id": "427fd58d6660abf6f38a_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"],\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "6db4ac493194a52cbd3a_hinglish", "retrieved_id": "25ac9041dfe1446dd870_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"],\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "0a2ec84ba0e594d94b9e_en", "retrieved_id": "ff73314a0982d6281641_noisy_en", "output": "{\"action\":\"call\",\"name\":\"database.describe_table\",\"arguments\":{\"table\":\"books\"}}", "metrics": {"json_valid": true, "schema_valid": false, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false, "error": "Unknown tool"}} +{"id": "0a2ec84ba0e594d94b9e_noisy_en", "retrieved_id": "03727992a863b145c565_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "0a2ec84ba0e594d94b9e_hi", "retrieved_id": "26ee7e71ccaee33a6162_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "0a2ec84ba0e594d94b9e_hinglish", "retrieved_id": "0af23c07619fdf39d636_hinglish", "output": "{\"action\":\"call\",\"name\":\"database.describe_table\",\"arguments\":{\"table\":\"books\"}}", "metrics": {"json_valid": true, "schema_valid": false, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false, "error": "Unknown tool"}} +{"id": "35b25e2b46b13ad30caf_en", "retrieved_id": "a5928e9d2a9de389c96e_noisy_en", "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "35b25e2b46b13ad30caf_noisy_en", "retrieved_id": "7b86419573eea700d641_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": false, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false}} +{"id": "35b25e2b46b13ad30caf_hi", "retrieved_id": "d8fbf98bcb77ca92a390_hi", "output": "{\"action\":\"clarify\",\"question\":\"सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "35b25e2b46b13ad30caf_hinglish", "retrieved_id": "981c50947db171d5fec5_hinglish", "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "916ff7ce3eea9aa64393_en", "retrieved_id": "f3e33fe8b4634a01b81b_noisy_en", "output": "{\"action\":\"clarify\",\"question\":\"Which value should kind equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "916ff7ce3eea9aa64393_noisy_en", "retrieved_id": "f3e33fe8b4634a01b81b_noisy_en", "output": "{\"action\":\"clarify\",\"question\":\"Which value should kind equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "916ff7ce3eea9aa64393_hi", "retrieved_id": "db76a8fce72687acde45_hi", "output": "{\"action\":\"clarify\",\"question\":\"kind का कौन सा मान चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "916ff7ce3eea9aa64393_hinglish", "retrieved_id": "6aaa43e87f1033fe0daa_hinglish", "output": "{\"action\":\"clarify\",\"question\":\"kind ki kaunsi value chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "49c2e66e48d062e5c69c_en", "retrieved_id": "8fde1669f8ae856382b5_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": false, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false}} +{"id": "49c2e66e48d062e5c69c_noisy_en", "retrieved_id": "f6d1979b8e8273fb994b_en", "output": "{\"action\":\"clarify\",\"question\":\"This connection is read-only. Would you like a SELECT query instead?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "49c2e66e48d062e5c69c_hi", "retrieved_id": "a4f18f2163d8129dfc3c_hi", "output": "{\"action\":\"clarify\",\"question\":\"यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "49c2e66e48d062e5c69c_hinglish", "retrieved_id": "6cad228416bb08b460f6_hinglish", "output": "{\"action\":\"clarify\",\"question\":\"Yeh connection read-only hai. Kya SELECT query chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "6600e72cfc86012212d0_en", "retrieved_id": "1173d51478c0f0770b25_en", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "6600e72cfc86012212d0_noisy_en", "retrieved_id": "8f78f2ccd606ac642613_noisy_en", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "6600e72cfc86012212d0_hi", "retrieved_id": "04fd2c1a67c04de23381_hi", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "6600e72cfc86012212d0_hinglish", "retrieved_id": "240cb7bac57363d0776e_hinglish", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "4e42bbd40467b2b8c27f_en", "retrieved_id": "33f141708d95562cf305_en", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"SQL joins\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "4e42bbd40467b2b8c27f_noisy_en", "retrieved_id": "7a10eeb0c23fcd9532e0_en", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"SQL joins\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "4e42bbd40467b2b8c27f_hi", "retrieved_id": "5d4eee92427273c1a849_hi", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"SQL joins\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "4e42bbd40467b2b8c27f_hinglish", "retrieved_id": "e1237888e640d85992e3_hinglish", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"SQL joins\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "b6a1ad15475eea3c8bfb_en", "retrieved_id": "db4e496cbbe068f26fe3_en", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/reports/sales.csv\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "b6a1ad15475eea3c8bfb_noisy_en", "retrieved_id": "db4e496cbbe068f26fe3_en", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/reports/sales.csv\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "b6a1ad15475eea3c8bfb_hi", "retrieved_id": "44623cd201dfc0eae051_hi", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/reports/sales.csv\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "b6a1ad15475eea3c8bfb_hinglish", "retrieved_id": "dba89384b2e426377857_hinglish", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/reports/sales.csv\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "cca31e425efaaadf4c97_en", "retrieved_id": "7433354d41a6c516f4b6_en", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-8757\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "cca31e425efaaadf4c97_noisy_en", "retrieved_id": "4c422d229cf597a3acdd_noisy_en_curriculum_99606", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-117\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "cca31e425efaaadf4c97_hi", "retrieved_id": "b0c31fd3cb1afd956208_hi", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-9402\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "cca31e425efaaadf4c97_hinglish", "retrieved_id": "6ea26747bfbf3b1aa5e0_hinglish", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-8757\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "40ffef08b12e4a3276e3_en", "retrieved_id": "f1fd879c85eb3cbe5abc_en", "output": "{\"action\":\"answer\",\"text\":\"The tool returned 116 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "40ffef08b12e4a3276e3_noisy_en", "retrieved_id": "f1fd879c85eb3cbe5abc_en", "output": "{\"action\":\"answer\",\"text\":\"The tool returned 116 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "40ffef08b12e4a3276e3_hi", "retrieved_id": "f1fd879c85eb3cbe5abc_hi", "output": "{\"action\":\"answer\",\"text\":\"टूल ने 116 पंक्तियाँ लौटाईं।\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "40ffef08b12e4a3276e3_hinglish", "retrieved_id": "f1fd879c85eb3cbe5abc_en", "output": "{\"action\":\"answer\",\"text\":\"The tool returned 116 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false, "text_exact": false}} +{"id": "c07bcd68edd3354fcd83_en", "retrieved_id": "130f7bdeb1d62731f52f_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c07bcd68edd3354fcd83_noisy_en", "retrieved_id": "88a8a9447d5602b34319_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c07bcd68edd3354fcd83_hi", "retrieved_id": "02501bf0d6866d9d8bf4_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c07bcd68edd3354fcd83_hinglish", "retrieved_id": "484e67e978b01025c01e_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "84b620e6d482597eea06_en", "retrieved_id": "427c52f3cda588a8dfa5_concrete_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT kind FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "84b620e6d482597eea06_noisy_en", "retrieved_id": "b2757aefcc8bc0a7549e_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "84b620e6d482597eea06_hi", "retrieved_id": "3aac2f1448f136869091_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "84b620e6d482597eea06_hinglish", "retrieved_id": "6eff07d389c9dffd14ce_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "42cda457b51f6f9d3056_en", "retrieved_id": "0d3d3c2d1d7825bb7ee8_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM inspections WHERE region = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "42cda457b51f6f9d3056_noisy_en", "retrieved_id": "0244585864374715cc77_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM inspections WHERE region = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "42cda457b51f6f9d3056_hi", "retrieved_id": "ffd58b72505e64c256d5_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE region = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "42cda457b51f6f9d3056_hinglish", "retrieved_id": "1c3073a518a1586baf2d_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE region = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "15b265bd8bbdb3515605_en", "retrieved_id": "4ea4127e4c100bc42d61_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT display_name FROM inspections WHERE kind = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "15b265bd8bbdb3515605_noisy_en", "retrieved_id": "4ea4127e4c100bc42d61_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT display_name FROM inspections WHERE kind = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "15b265bd8bbdb3515605_hi", "retrieved_id": "4ea4127e4c100bc42d61_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT display_name FROM inspections WHERE kind = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "15b265bd8bbdb3515605_hinglish", "retrieved_id": "99f1fbfee69979f7cbc0_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT display_name FROM inspections WHERE kind = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4a79981adf7647bac6b9_en", "retrieved_id": "1b3dd8c5ea0aa5d4644b_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE amount <= 500;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "4a79981adf7647bac6b9_noisy_en", "retrieved_id": "f27668acc136794b63af_concrete_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE amount > 10000;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "4a79981adf7647bac6b9_hi", "retrieved_id": "93630964314d2a1070ff_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE description IS NULL;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "4a79981adf7647bac6b9_hinglish", "retrieved_id": "f2e8bb8ff7f8c02b0623_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE amount < 500;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "d5fc4318c9aedb495460_en", "retrieved_id": "23100cb2db54f16447a9_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE price < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d5fc4318c9aedb495460_noisy_en", "retrieved_id": "0d87e4597b8030db1d39_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE price < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d5fc4318c9aedb495460_hi", "retrieved_id": "6c50e81ff0658a9ae684_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE price < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d5fc4318c9aedb495460_hinglish", "retrieved_id": "d0e599f5b10bb2a670bd_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE price < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e8f90532d34c12209a83_en", "retrieved_id": "fe0c250a5b61d276b6ce_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE salary > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "e8f90532d34c12209a83_noisy_en", "retrieved_id": "18b58f4b6324bf5e24ef_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE salary < 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "e8f90532d34c12209a83_hi", "retrieved_id": "df2bcb22f6685fc818a1_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE salary < 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "e8f90532d34c12209a83_hinglish", "retrieved_id": "9a18682fc49a50e8ab0f_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE salary >= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0352841d74f7bd7955ea_en", "retrieved_id": "a05483c4e8aab9de17d6_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE department = 'Mumbai' AND score > 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "0352841d74f7bd7955ea_noisy_en", "retrieved_id": "20dd274ccc92b3dd7942_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE score <= 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0352841d74f7bd7955ea_hi", "retrieved_id": "442e0f59c0a47acce280_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE score >= 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "0352841d74f7bd7955ea_hinglish", "retrieved_id": "13da1fb7f3c2c4be3be5_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE score <= 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "645fd36c186709e64922_en", "retrieved_id": "652861e0ccdcf2bb7315_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE amount BETWEEN 1000 AND 1010;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "645fd36c186709e64922_noisy_en", "retrieved_id": "64e3df4343ec2aae7140_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE amount BETWEEN 500 AND 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "645fd36c186709e64922_hi", "retrieved_id": "652861e0ccdcf2bb7315_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE amount BETWEEN 1000 AND 1010;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "645fd36c186709e64922_hinglish", "retrieved_id": "652861e0ccdcf2bb7315_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE amount BETWEEN 1000 AND 1010;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "70da7b441b3a995da19a_en", "retrieved_id": "f1192c9734e97976d459_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE city = 'O''Reilly' AND quantity > 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "70da7b441b3a995da19a_noisy_en", "retrieved_id": "b3d9599cb92dc707b13f_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE city = 'O''Reilly' AND quantity > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "70da7b441b3a995da19a_hi", "retrieved_id": "b3d9599cb92dc707b13f_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE city = 'O''Reilly' AND quantity > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "70da7b441b3a995da19a_hinglish", "retrieved_id": "b3d9599cb92dc707b13f_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE city = 'O''Reilly' AND quantity > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bcd7a532a06274ddddee_en", "retrieved_id": "7d85dce1914021369d8c_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE status IN ('pending', 'South');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bcd7a532a06274ddddee_noisy_en", "retrieved_id": "7d85dce1914021369d8c_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE status IN ('pending', 'South');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bcd7a532a06274ddddee_hi", "retrieved_id": "afb47c9f3de8df97b7e7_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE status IN ('South', 'pending');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "bcd7a532a06274ddddee_hinglish", "retrieved_id": "7d85dce1914021369d8c_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE status IN ('pending', 'South');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "04b2e5886e750df09a51_en", "retrieved_id": "708107568853e23d7fbe_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE display_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "04b2e5886e750df09a51_noisy_en", "retrieved_id": "c9ac52ed487c1889af4f_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE display_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "04b2e5886e750df09a51_hi", "retrieved_id": "504051dee53dda1de46e_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE NOT display_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "04b2e5886e750df09a51_hinglish", "retrieved_id": "b37da50864de15a3c7f6_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE display_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "060573c57829f3ce6e8f_en", "retrieved_id": "eaefffbb9d33d7bc6a06_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE item_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "060573c57829f3ce6e8f_noisy_en", "retrieved_id": "7598902252e29976110f_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "060573c57829f3ce6e8f_hi", "retrieved_id": "bdce576673636f7b523c_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE NOT item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "060573c57829f3ce6e8f_hinglish", "retrieved_id": "eaefffbb9d33d7bc6a06_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE item_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d61a27de98bb7d44f15d_en", "retrieved_id": "c40602913c502e7ccda9_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM specimens WHERE department = 'Mumbai';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "d61a27de98bb7d44f15d_noisy_en", "retrieved_id": "6040c6883947ec5b3821_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d61a27de98bb7d44f15d_hi", "retrieved_id": "2d416c705002edc7a1af_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d61a27de98bb7d44f15d_hinglish", "retrieved_id": "33e0077bee491c810f9d_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "fea44ee93f1cea0883dd_en", "retrieved_id": "cbc3795c0ca21a9b8c5a_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "fea44ee93f1cea0883dd_noisy_en", "retrieved_id": "ae83fec5fe607c748ed4_concrete_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM exhibits WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "fea44ee93f1cea0883dd_hi", "retrieved_id": "e14ee545219a2fdae9e2_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE category IN ('Delhi', 'Delhi');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "fea44ee93f1cea0883dd_hinglish", "retrieved_id": "ae83fec5fe607c748ed4_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM exhibits WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d30264bbc3b08aedbc51_en", "retrieved_id": "feb1c00cb03f307577c4_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT city, SUM(score) FROM exhibits GROUP BY city;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "d30264bbc3b08aedbc51_noisy_en", "retrieved_id": "0cc071ba9b202685fdea_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT SUM(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d30264bbc3b08aedbc51_hi", "retrieved_id": "fea4614c3df958e07226_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT SUM(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d30264bbc3b08aedbc51_hinglish", "retrieved_id": "b30ba6196f2ca49779e7_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT SUM(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "64372a29664a7eaddbe2_en", "retrieved_id": "631610dd615b5944c53c_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT AVG(amount) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "64372a29664a7eaddbe2_noisy_en", "retrieved_id": "af8b9567686a7bea4c9b_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT AVG(amount) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "64372a29664a7eaddbe2_hi", "retrieved_id": "4d38dd14f889658dbb33_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT AVG(amount) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "64372a29664a7eaddbe2_hinglish", "retrieved_id": "d736aa1e5b28a83ca247_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT AVG(amount) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6f28c044459ce99cf405_en", "retrieved_id": "747ca8e86fd928154f9c_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT MAX(quantity) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6f28c044459ce99cf405_noisy_en", "retrieved_id": "e58b057a43bf930f66c1_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT MAX(quantity) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6f28c044459ce99cf405_hi", "retrieved_id": "b6185f615e05ea951f2f_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT SUM(quantity) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "6f28c044459ce99cf405_hinglish", "retrieved_id": "8791137d1c7a2875f0db_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT MIN(quantity) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "c0b9cf3b44d96839f256_en", "retrieved_id": "f087c53ca6b81612fdc9_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(balance) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c0b9cf3b44d96839f256_noisy_en", "retrieved_id": "e7cf7fc9a897e03057a6_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(balance) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c0b9cf3b44d96839f256_hi", "retrieved_id": "f087c53ca6b81612fdc9_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(balance) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c0b9cf3b44d96839f256_hinglish", "retrieved_id": "960d323d801fb040b366_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(balance) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7dd163238936d96d0bd4_en", "retrieved_id": "0a760c859ef8f38c0243_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT city FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7dd163238936d96d0bd4_noisy_en", "retrieved_id": "0a760c859ef8f38c0243_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT city FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7dd163238936d96d0bd4_hi", "retrieved_id": "c29770098dbbf4c38518_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT city FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7dd163238936d96d0bd4_hinglish", "retrieved_id": "f0e8a1aa17ebe2943f9a_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "4e9184b509f43a710021_en", "retrieved_id": "b51fb5d4346bf52d0b55_concrete_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY CASE WHEN balance IS NULL THEN 1 ELSE 0 END DESC, balance DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "4e9184b509f43a710021_noisy_en", "retrieved_id": "1bd4bf74ba12b81ef4b7_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY balance DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4e9184b509f43a710021_hi", "retrieved_id": "1bd4bf74ba12b81ef4b7_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY balance DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4e9184b509f43a710021_hinglish", "retrieved_id": "9360abd24d181d80b919_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY balance DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5aaaf945b8fba37eadf4_en", "retrieved_id": "36010b0cd2ebc005b766_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY CASE WHEN price IS NULL THEN 1 ELSE 0 END DESC, price DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "5aaaf945b8fba37eadf4_noisy_en", "retrieved_id": "36010b0cd2ebc005b766_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY CASE WHEN price IS NULL THEN 1 ELSE 0 END DESC, price DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "5aaaf945b8fba37eadf4_hi", "retrieved_id": "6a30e3aab87d8b8fac24_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY CASE WHEN price IS NULL THEN 1 ELSE 0 END, price ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "5aaaf945b8fba37eadf4_hinglish", "retrieved_id": "d49736acab21961ee1c4_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY price ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "fd0bab1cc08d7298f131_en", "retrieved_id": "a6540b5f2c0929241f31_concrete_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY salary ASC NULLS FIRST LIMIT 20;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "fd0bab1cc08d7298f131_noisy_en", "retrieved_id": "930384317e1465fd3f65_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC NULLS LAST LIMIT 20;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "fd0bab1cc08d7298f131_hi", "retrieved_id": "5abb5a811736206c4a4c_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC NULLS LAST LIMIT 20;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "fd0bab1cc08d7298f131_hinglish", "retrieved_id": "8dc7bed1546d4cec4da6_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC NULLS LAST LIMIT 20;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "a29fdee3339da6711080_en", "retrieved_id": "797a7cee699e825b8d9d_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY CASE WHEN rating IS NULL THEN 1 ELSE 0 END, rating ASC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "a29fdee3339da6711080_noisy_en", "retrieved_id": "797a7cee699e825b8d9d_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY CASE WHEN rating IS NULL THEN 1 ELSE 0 END, rating ASC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "a29fdee3339da6711080_hi", "retrieved_id": "3a5ecd2a0421c0799bf9_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY CASE WHEN rating IS NULL THEN 1 ELSE 0 END, rating ASC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "a29fdee3339da6711080_hinglish", "retrieved_id": "797a7cee699e825b8d9d_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY CASE WHEN rating IS NULL THEN 1 ELSE 0 END, rating ASC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "92e36c8a6ca6e7aa2298_en", "retrieved_id": "39ffd26f283fc2242233_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "92e36c8a6ca6e7aa2298_noisy_en", "retrieved_id": "6ad175c1da38937a1c73_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "92e36c8a6ca6e7aa2298_hi", "retrieved_id": "11be592bce7c6de29c95_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "92e36c8a6ca6e7aa2298_hinglish", "retrieved_id": "c2e6ba167b989ceb36a1_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "61b3db48f1ff12dc69e1_en", "retrieved_id": "7cc4fdc4c1c250b89bee_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT kind, SUM(quantity) FROM exhibits GROUP BY kind;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "61b3db48f1ff12dc69e1_noisy_en", "retrieved_id": "287ddb0a62835bf72fc2_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT kind, SUM(quantity) FROM exhibits GROUP BY kind;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "61b3db48f1ff12dc69e1_hi", "retrieved_id": "56d7e4f92c2de0a646af_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT kind, SUM(quantity) FROM exhibits GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "61b3db48f1ff12dc69e1_hinglish", "retrieved_id": "f7b38dbaa058adc9656d_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT kind, SUM(quantity) FROM exhibits GROUP BY kind;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9ebca064e63b0347236b_en", "retrieved_id": "97445a75f67055563e4b_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 10;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "9ebca064e63b0347236b_noisy_en", "retrieved_id": "3a27f3b2078ae2fa7bd5_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 5;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9ebca064e63b0347236b_hi", "retrieved_id": "73cd027b6dc2b21877ec_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 500;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "9ebca064e63b0347236b_hinglish", "retrieved_id": "c72390855f38f5cde5dd_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 5;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "85e339da2fb28bae5395_en", "retrieved_id": "c209f35bb753c4c6c4c8_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE EXTRACT(YEAR FROM order_date) = 2024;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "85e339da2fb28bae5395_noisy_en", "retrieved_id": "e729cceb6c7d2457bb92_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE order_date >= '2024-11-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "85e339da2fb28bae5395_hi", "retrieved_id": "23419d787a8f4dd33ff3_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE order_date >= '2026-10-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "85e339da2fb28bae5395_hinglish", "retrieved_id": "da1047341b9a4ac08169_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE order_date >= '2024-10-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8a456280df4a711c3cda_en", "retrieved_id": "fa668f5f74b7998e4689_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE EXTRACT(YEAR FROM CAST(created_at AS DATE)) = 2024;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "8a456280df4a711c3cda_noisy_en", "retrieved_id": "774eabfe1a00326f842e_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE EXTRACT(YEAR FROM CAST(created_at AS DATE)) = 2024;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "8a456280df4a711c3cda_hi", "retrieved_id": "52c736038a7903fa3560_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE EXTRACT(YEAR FROM CAST(created_at AS DATE)) = 2024;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "8a456280df4a711c3cda_hinglish", "retrieved_id": "4a29dc069452fce48052_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE EXTRACT(YEAR FROM created_at) = 2024;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2e9e0c695ac50e3982d7_en", "retrieved_id": "1168a6c1687df2e3cf7b_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE order_date >= '2022-12-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "2e9e0c695ac50e3982d7_noisy_en", "retrieved_id": "cbe93f0a810e7b8aeae7_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE EXTRACT(MONTH FROM order_date) = 12;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "2e9e0c695ac50e3982d7_hi", "retrieved_id": "187a269ce9db7a832c10_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE EXTRACT(MONTH FROM order_date) = 12;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "2e9e0c695ac50e3982d7_hinglish", "retrieved_id": "cbe93f0a810e7b8aeae7_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE EXTRACT(MONTH FROM order_date) = 12;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "e3d12f1bf51bbf54a331_en", "retrieved_id": "7d1ac8d4b0011bdc8474_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE LOWER(description) LIKE LOWER('%North%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e3d12f1bf51bbf54a331_noisy_en", "retrieved_id": "279c576638ef45226927_concrete_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE LOWER(description) LIKE LOWER('%Books%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "e3d12f1bf51bbf54a331_hi", "retrieved_id": "0a2568c82cff05606439_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE LOWER(description) LIKE LOWER('%North%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e3d12f1bf51bbf54a331_hinglish", "retrieved_id": "072d96270b89757b7dd0_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE LOWER(description) LIKE LOWER('%North%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "307d5b7bd93949f7381a_en", "retrieved_id": "22701200b58e29b4c0ab_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "307d5b7bd93949f7381a_noisy_en", "retrieved_id": "22701200b58e29b4c0ab_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "307d5b7bd93949f7381a_hi", "retrieved_id": "949b6e0ae2bd47e7b2a2_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "307d5b7bd93949f7381a_hinglish", "retrieved_id": "5f940b0e9d7dfa863d6b_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a7675c74645d63d69aac_en", "retrieved_id": "9a6b6814886a204703d3_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT reservations.description, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "a7675c74645d63d69aac_noisy_en", "retrieved_id": "64ff1bce974662a9d8af_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'pending';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a7675c74645d63d69aac_hi", "retrieved_id": "906c201d50d4211d49bf_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'pending';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a7675c74645d63d69aac_hinglish", "retrieved_id": "0b2aba21b33459b51950_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'pending';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "aa4f0e1f29573e237ee1_en", "retrieved_id": "bd6b78905a3ad03e04b6_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specimens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "aa4f0e1f29573e237ee1_noisy_en", "retrieved_id": "bd6b78905a3ad03e04b6_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specimens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "aa4f0e1f29573e237ee1_hi", "retrieved_id": "944772e48c9b5194fe1e_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specimens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "aa4f0e1f29573e237ee1_hinglish", "retrieved_id": "5165d04cc5f4626621a5_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"],\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "4fb8670235a435c82c88_en", "retrieved_id": "863d3ef71bbbeba4919e_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"],\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "4fb8670235a435c82c88_noisy_en", "retrieved_id": "632bfa2e24875823150a_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "4fb8670235a435c82c88_hi", "retrieved_id": "286869979f46e9bbc61c_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"],\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "4fb8670235a435c82c88_hinglish", "retrieved_id": "4253da29b10c8d2d921b_hinglish", "output": "{\"action\":\"call\",\"name\":\"describe_table\",\"arguments\":{\"table\":\"books\"}}", "metrics": {"json_valid": true, "schema_valid": false, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false, "error": "Unknown tool"}} +{"id": "5cb3a450bcdc957b7ba5_en", "retrieved_id": "fb890425dce9de189858_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "5cb3a450bcdc957b7ba5_noisy_en", "retrieved_id": "25ac9041dfe1446dd870_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "5cb3a450bcdc957b7ba5_hi", "retrieved_id": "427fd58d6660abf6f38a_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "5cb3a450bcdc957b7ba5_hinglish", "retrieved_id": "25ac9041dfe1446dd870_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "3b3a8e441f0ae5cbaad4_en", "retrieved_id": "a5928e9d2a9de389c96e_noisy_en", "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "3b3a8e441f0ae5cbaad4_noisy_en", "retrieved_id": "7b86419573eea700d641_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": false, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false}} +{"id": "3b3a8e441f0ae5cbaad4_hi", "retrieved_id": "d8fbf98bcb77ca92a390_hi", "output": "{\"action\":\"clarify\",\"question\":\"सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "3b3a8e441f0ae5cbaad4_hinglish", "retrieved_id": "981c50947db171d5fec5_hinglish", "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "ae26d8175253b650bb6b_en", "retrieved_id": "a44004f0eb7cc5872af0_noisy_en", "output": "{\"action\":\"clarify\",\"question\":\"Which value should region equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "ae26d8175253b650bb6b_noisy_en", "retrieved_id": "a44004f0eb7cc5872af0_noisy_en", "output": "{\"action\":\"clarify\",\"question\":\"Which value should region equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "ae26d8175253b650bb6b_hi", "retrieved_id": "90ab6a4e7d8fa9ce50fb_hi", "output": "{\"action\":\"clarify\",\"question\":\"region का कौन सा मान चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "ae26d8175253b650bb6b_hinglish", "retrieved_id": "8c963f079269e7395aa1_hinglish", "output": "{\"action\":\"clarify\",\"question\":\"region ki kaunsi value chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "2984a6707d4e07f81695_en", "retrieved_id": "8fde1669f8ae856382b5_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": false, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false}} +{"id": "2984a6707d4e07f81695_noisy_en", "retrieved_id": "f6d1979b8e8273fb994b_en", "output": "{\"action\":\"clarify\",\"question\":\"This connection is read-only. Would you like a SELECT query instead?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "2984a6707d4e07f81695_hi", "retrieved_id": "a4f18f2163d8129dfc3c_hi", "output": "{\"action\":\"clarify\",\"question\":\"यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "2984a6707d4e07f81695_hinglish", "retrieved_id": "6cad228416bb08b460f6_hinglish", "output": "{\"action\":\"clarify\",\"question\":\"Yeh connection read-only hai. Kya SELECT query chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "b5839920055891fb68c8_en", "retrieved_id": "60c088bfa4c1573171ce_en", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Lucknow\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "b5839920055891fb68c8_noisy_en", "retrieved_id": "19be7f84e5c1fc8bab22_noisy_en", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Lucknow\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "b5839920055891fb68c8_hi", "retrieved_id": "19be7f84e5c1fc8bab22_hi", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Lucknow\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "b5839920055891fb68c8_hinglish", "retrieved_id": "c8617cfd9655faabcf9e_hinglish", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Lucknow\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "adad513dde5be462c95d_en", "retrieved_id": "f774a686ab7457235c97_en", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"row level security\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "adad513dde5be462c95d_noisy_en", "retrieved_id": "e68655c8c85fca3e9600_en", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"row level security\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "adad513dde5be462c95d_hi", "retrieved_id": "653957df1273598cdf74_hi", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"row level security\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "adad513dde5be462c95d_hinglish", "retrieved_id": "344d182855c89b4a86b2_hinglish", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"row level security\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "6f3055c580b33340c854_en", "retrieved_id": "9c9565eadc46c39ab34c_en", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "6f3055c580b33340c854_noisy_en", "retrieved_id": "9c9565eadc46c39ab34c_en", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "6f3055c580b33340c854_hi", "retrieved_id": "00319aa9372ffea135d7_hi", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "6f3055c580b33340c854_hinglish", "retrieved_id": "d53f5c6036055adc1894_hinglish", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "5ebc52b3d20bde2076b3_en", "retrieved_id": "c745f3dc5489adf68488_noisy_en", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6124\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "5ebc52b3d20bde2076b3_noisy_en", "retrieved_id": "c745f3dc5489adf68488_noisy_en", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6124\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "5ebc52b3d20bde2076b3_hi", "retrieved_id": "c745f3dc5489adf68488_hi", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6124\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "5ebc52b3d20bde2076b3_hinglish", "retrieved_id": "c745f3dc5489adf68488_noisy_en", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6124\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "f55358178d003de7ba56_en", "retrieved_id": "2cf30f6c9041428c9fa8_noisy_en", "output": "{\"action\":\"answer\",\"text\":\"The tool returned 133 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "f55358178d003de7ba56_noisy_en", "retrieved_id": "2cf30f6c9041428c9fa8_noisy_en", "output": "{\"action\":\"answer\",\"text\":\"The tool returned 133 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "f55358178d003de7ba56_hi", "retrieved_id": "82152ba010f308114e05_hi", "output": "{\"action\":\"answer\",\"text\":\"टूल ने 133 पंक्तियाँ लौटाईं।\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "f55358178d003de7ba56_hinglish", "retrieved_id": "2cf30f6c9041428c9fa8_hinglish", "output": "{\"action\":\"answer\",\"text\":\"Tool ne 133 rows return ki.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "6d9e81a9f52b8d33060a_en", "retrieved_id": "130f7bdeb1d62731f52f_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6d9e81a9f52b8d33060a_noisy_en", "retrieved_id": "88a8a9447d5602b34319_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6d9e81a9f52b8d33060a_hi", "retrieved_id": "02501bf0d6866d9d8bf4_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6d9e81a9f52b8d33060a_hinglish", "retrieved_id": "484e67e978b01025c01e_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c86deada03a046468575_en", "retrieved_id": "427c52f3cda588a8dfa5_concrete_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT DISTINCT region FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "c86deada03a046468575_noisy_en", "retrieved_id": "f158884fc953e9bd6dc7_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c86deada03a046468575_hi", "retrieved_id": "3aac2f1448f136869091_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT item_name FROM reservations;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c86deada03a046468575_hinglish", "retrieved_id": "1e5a4a69bda180f02aa2_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c15c59356c7863f0b7e9_en", "retrieved_id": "464d252657fc1d8b618b_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "c15c59356c7863f0b7e9_noisy_en", "retrieved_id": "fe6ca9f2c1325acb8042_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens WHERE city = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "c15c59356c7863f0b7e9_hi", "retrieved_id": "7818c3c03d0bb1143d82_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE city = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c15c59356c7863f0b7e9_hinglish", "retrieved_id": "f5686907eddcf41fc4c4_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE city = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "cfb6d9bde68e2cdc5bf9_en", "retrieved_id": "25312378405a2b658fc0_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "cfb6d9bde68e2cdc5bf9_noisy_en", "retrieved_id": "14657d9528faa891de0c_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE category = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "cfb6d9bde68e2cdc5bf9_hi", "retrieved_id": "872ed9328d433c3ac120_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT name FROM reservations WHERE category = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "cfb6d9bde68e2cdc5bf9_hinglish", "retrieved_id": "872ed9328d433c3ac120_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT name FROM reservations WHERE category = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b45bf4e81303e2b4da0f_en", "retrieved_id": "19dcf10064ff83f34082_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE price <= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "b45bf4e81303e2b4da0f_noisy_en", "retrieved_id": "f27668acc136794b63af_concrete_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE price > 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "b45bf4e81303e2b4da0f_hi", "retrieved_id": "93630964314d2a1070ff_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE customer_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "b45bf4e81303e2b4da0f_hinglish", "retrieved_id": "b0b6ce977a821d812fab_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE price < 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "0294353d6bde2cd5185c_en", "retrieved_id": "a5e151edfd6624692e8e_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE cost < 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0294353d6bde2cd5185c_noisy_en", "retrieved_id": "a20e0dc7c0250b340c33_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE cost < 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0294353d6bde2cd5185c_hi", "retrieved_id": "e937c5ad7e6337dc56b2_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE cost < 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0294353d6bde2cd5185c_hinglish", "retrieved_id": "e937c5ad7e6337dc56b2_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE cost < 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "96a74866d51ea7899cd8_en", "retrieved_id": "7abd964797f7e52924be_concrete_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE score > 5000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "96a74866d51ea7899cd8_noisy_en", "retrieved_id": "b1caec5d45240e74355d_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE score < 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "96a74866d51ea7899cd8_hi", "retrieved_id": "4cdbd72a8fb19b994edf_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE score < 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "96a74866d51ea7899cd8_hinglish", "retrieved_id": "880bca85abb064cea42d_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE score >= 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9d0222589e9977ea3f57_en", "retrieved_id": "9cda89f9807faf90c36f_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE balance > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "9d0222589e9977ea3f57_noisy_en", "retrieved_id": "57463a9d8124deb9b65e_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE balance <= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9d0222589e9977ea3f57_hi", "retrieved_id": "02eb13e187597ae5f52c_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE balance >= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "9d0222589e9977ea3f57_hinglish", "retrieved_id": "0f6246ae71d84c63697d_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE balance <= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5f0b5d842746b96c126b_en", "retrieved_id": "781215fe9ee9b6f17c65_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price BETWEEN 1000 AND 1100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5f0b5d842746b96c126b_noisy_en", "retrieved_id": "f9aaaf19a1e69b3cf5d4_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price BETWEEN 1000 AND 1100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5f0b5d842746b96c126b_hi", "retrieved_id": "781215fe9ee9b6f17c65_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price BETWEEN 1000 AND 1100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5f0b5d842746b96c126b_hinglish", "retrieved_id": "54be1eb979e2f7e06cec_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price BETWEEN 1000 AND 1100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "864b7a7bf739071bb030_en", "retrieved_id": "abf04d4be26387be55d8_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE status = 'Pune' AND balance > 0;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "864b7a7bf739071bb030_noisy_en", "retrieved_id": "c9395756638c1e205151_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE status = 'Pune' AND balance > 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "864b7a7bf739071bb030_hi", "retrieved_id": "abf04d4be26387be55d8_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE status = 'Pune' AND balance > 0;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "864b7a7bf739071bb030_hinglish", "retrieved_id": "d75307338240fe27d93e_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE status = 'Pune' AND balance > 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "fd2a795a1557dc1487c6_en", "retrieved_id": "2f359063c9c22b0bd46d_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE city IN ('Electronics', 'Pune');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "fd2a795a1557dc1487c6_noisy_en", "retrieved_id": "65a054a18b6c12d0d7b2_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE city IN ('Pune', 'Electronics');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "fd2a795a1557dc1487c6_hi", "retrieved_id": "372922de4f76aa6f8ca6_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE city IN ('Electronics', 'Pune');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "fd2a795a1557dc1487c6_hinglish", "retrieved_id": "65a054a18b6c12d0d7b2_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE city IN ('Pune', 'Electronics');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "208e9ccf17a923a46aac_en", "retrieved_id": "b938809a21ca895bfdaa_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "208e9ccf17a923a46aac_noisy_en", "retrieved_id": "999181602f630fe41f53_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "208e9ccf17a923a46aac_hi", "retrieved_id": "f1b3784f5142d5df00b8_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE item_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "208e9ccf17a923a46aac_hinglish", "retrieved_id": "c39505a9b595f6ac6b82_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "3ad8170567694db8c8f9_en", "retrieved_id": "c7bb7b4dfc547acf0b9c_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE display_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "3ad8170567694db8c8f9_noisy_en", "retrieved_id": "348951272c18ab797a78_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE display_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "3ad8170567694db8c8f9_hi", "retrieved_id": "c7bb7b4dfc547acf0b9c_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE display_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "3ad8170567694db8c8f9_hinglish", "retrieved_id": "1bbb8e0920a5610a79fb_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE display_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "55cd623d82474692b1a9_en", "retrieved_id": "c40602913c502e7ccda9_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens WHERE category = 'Mumbai';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "55cd623d82474692b1a9_noisy_en", "retrieved_id": "6040c6883947ec5b3821_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "55cd623d82474692b1a9_hi", "retrieved_id": "2d416c705002edc7a1af_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "55cd623d82474692b1a9_hinglish", "retrieved_id": "33e0077bee491c810f9d_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e2b48c4ec5252934cb20_en", "retrieved_id": "cbc3795c0ca21a9b8c5a_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "e2b48c4ec5252934cb20_noisy_en", "retrieved_id": "8463566151edd133626c_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM inspections WHERE city = 'pending';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e2b48c4ec5252934cb20_hi", "retrieved_id": "60a34656f2f657c68d8a_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE city IN ('pending', 'pending');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "e2b48c4ec5252934cb20_hinglish", "retrieved_id": "8f46a1742f10c08d6c17_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM inspections WHERE city = 'pending';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1110104f12e10c5a007a_en", "retrieved_id": "e2f04ba7741b597ad65e_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"],\"project_id\":\"demo_inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": false, "success": false, "error": "'query'"}} +{"id": "1110104f12e10c5a007a_noisy_en", "retrieved_id": "c81229be33656d74966c_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(amount) FROM inspections;\",\"project_id\":\"demo_inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1110104f12e10c5a007a_hi", "retrieved_id": "024f015b15d9be56d189_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT SUM(amount) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1110104f12e10c5a007a_hinglish", "retrieved_id": "de01d8d3700f90f01090_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(amount) FROM inspections;\",\"project_id\":\"demo_inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "772353a7da7eebcf3244_en", "retrieved_id": "f7a4253c6fd828fe0eb5_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(quantity) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "772353a7da7eebcf3244_noisy_en", "retrieved_id": "1ac79a7c76149227c94f_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(quantity) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "772353a7da7eebcf3244_hi", "retrieved_id": "77326d285145f100de34_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(quantity) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "772353a7da7eebcf3244_hinglish", "retrieved_id": "1ac79a7c76149227c94f_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(quantity) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8d2022223833a85bd2d1_en", "retrieved_id": "7cad2ba59a27227cbdd1_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT MAX(salary) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8d2022223833a85bd2d1_noisy_en", "retrieved_id": "4809982645208f5dadff_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT MAX(salary) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8d2022223833a85bd2d1_hi", "retrieved_id": "5e2e49adf71668e6010b_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT MIN(salary) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "8d2022223833a85bd2d1_hinglish", "retrieved_id": "2e4568fbee327c5f3f8e_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT MIN(salary) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "bc8593644a72fe8b9c78_en", "retrieved_id": "5ccb45f98330833db99f_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bc8593644a72fe8b9c78_noisy_en", "retrieved_id": "cd96f78f15f8210b1275_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bc8593644a72fe8b9c78_hi", "retrieved_id": "0d1a7f6b73322040214e_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bc8593644a72fe8b9c78_hinglish", "retrieved_id": "822cba9f4015c651b82c_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "69ac9800bd7a3ee23ccc_en", "retrieved_id": "c6325f377e0c7f0468f1_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT category FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "69ac9800bd7a3ee23ccc_noisy_en", "retrieved_id": "c6325f377e0c7f0468f1_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT category FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "69ac9800bd7a3ee23ccc_hi", "retrieved_id": "ce1d19d1f1e239c03e1b_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT category FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "69ac9800bd7a3ee23ccc_hinglish", "retrieved_id": "f0e8a1aa17ebe2943f9a_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "8f9ab277b699f2cbe988_en", "retrieved_id": "eae5287706bec5ee520e_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY CASE WHEN score IS NULL THEN 1 ELSE 0 END DESC, score DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "8f9ab277b699f2cbe988_noisy_en", "retrieved_id": "955a7d5f0d3cb038c3fa_concrete_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY score ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "8f9ab277b699f2cbe988_hi", "retrieved_id": "190107647feb114e3236_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY score DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8f9ab277b699f2cbe988_hinglish", "retrieved_id": "9360abd24d181d80b919_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY score DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ba705525122da37a7146_en", "retrieved_id": "56cddde09b6ce33e29af_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY CASE WHEN cost IS NULL THEN 1 ELSE 0 END DESC, cost DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "ba705525122da37a7146_noisy_en", "retrieved_id": "fd6c642df54b72b5146f_concrete_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY cost ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ba705525122da37a7146_hi", "retrieved_id": "3de06ebc9c472cfcc5f5_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY CASE WHEN cost IS NULL THEN 1 ELSE 0 END, cost ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "ba705525122da37a7146_hinglish", "retrieved_id": "0f074b2420cf3783035e_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY cost ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b7703ebecadd9433c5f8_en", "retrieved_id": "955a7d5f0d3cb038c3fa_concrete_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY quantity ASC NULLS FIRST;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "b7703ebecadd9433c5f8_noisy_en", "retrieved_id": "1b2780774c2ef3389909_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY quantity DESC LIMIT 3;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b7703ebecadd9433c5f8_hi", "retrieved_id": "36a2047a9b8d5e928dc2_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY quantity DESC NULLS LAST LIMIT 3;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "b7703ebecadd9433c5f8_hinglish", "retrieved_id": "2d0bbdceb628c88c79ac_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY quantity DESC NULLS LAST LIMIT 3;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "4e4b83d8af9eb4c5efc8_en", "retrieved_id": "75d72b4fdb50deab36a6_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY CASE WHEN amount IS NULL THEN 1 ELSE 0 END, amount ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "4e4b83d8af9eb4c5efc8_noisy_en", "retrieved_id": "75d72b4fdb50deab36a6_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY CASE WHEN amount IS NULL THEN 1 ELSE 0 END, amount ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "4e4b83d8af9eb4c5efc8_hi", "retrieved_id": "635fb6bbd3087bbcff95_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY CASE WHEN amount IS NULL THEN 1 ELSE 0 END, amount ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "4e4b83d8af9eb4c5efc8_hinglish", "retrieved_id": "b5899f88124102e83a64_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "039280d3483e998ccd1b_en", "retrieved_id": "2fe2748b6dc2b31719a4_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT department, COUNT(*) FROM reservations GROUP BY department;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "039280d3483e998ccd1b_noisy_en", "retrieved_id": "0ed04804842617d63993_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT department, COUNT(*) FROM reservations GROUP BY department;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "039280d3483e998ccd1b_hi", "retrieved_id": "cbcd01ed06edc207269b_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT department, COUNT(*) FROM reservations GROUP BY department;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "039280d3483e998ccd1b_hinglish", "retrieved_id": "7f0a16123b0e92f23bef_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT department, COUNT(*) FROM reservations GROUP BY department;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d96328fdb5748b255271_en", "retrieved_id": "38ff303a81302f75b326_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT category, SUM(salary) FROM inspections GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d96328fdb5748b255271_noisy_en", "retrieved_id": "8db52bac8a3a32a185cc_concrete_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT category, SUM(salary) FROM inspections GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d96328fdb5748b255271_hi", "retrieved_id": "edf1dd5e0f801cf61f15_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT category, SUM(salary) FROM inspections GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d96328fdb5748b255271_hinglish", "retrieved_id": "18410628daeae10df675_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT category, SUM(salary) FROM inspections GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "375b86b5cd0d00cd7868_en", "retrieved_id": "2a227c56316d84e5013b_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 2;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "375b86b5cd0d00cd7868_noisy_en", "retrieved_id": "45961aa2576646c7c61a_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 2;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "375b86b5cd0d00cd7868_hi", "retrieved_id": "94fcf554dae9ea314b84_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 2;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "375b86b5cd0d00cd7868_hinglish", "retrieved_id": "84cb2ac5c327df54d84d_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 2;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f1f8414c2d47f5396d69_en", "retrieved_id": "83e85315ccce26a9297f_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE created_at >= '2023-03-15';\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f1f8414c2d47f5396d69_noisy_en", "retrieved_id": "83e85315ccce26a9297f_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE created_at >= '2023-03-15';\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f1f8414c2d47f5396d69_hi", "retrieved_id": "83e85315ccce26a9297f_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE created_at >= '2023-03-15';\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f1f8414c2d47f5396d69_hinglish", "retrieved_id": "6be69c55f10ec1afec88_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE created_at >= '2025-03-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "5b5b87a5a58feab9d244_en", "retrieved_id": "b4ae57998c341af42244_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE EXTRACT(YEAR FROM CAST(recorded_at AS DATE)) = 2021;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "5b5b87a5a58feab9d244_noisy_en", "retrieved_id": "9f9c0a9aeabb87813004_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE EXTRACT(YEAR FROM recorded_at) = 2021;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5b5b87a5a58feab9d244_hi", "retrieved_id": "01cdb123039aea741782_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE EXTRACT(YEAR FROM recorded_at) = 2021;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5b5b87a5a58feab9d244_hinglish", "retrieved_id": "af1531839f3945723f77_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE EXTRACT(YEAR FROM CAST(recorded_at AS DATE)) = 2021;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "e8cfdfba296bc283222a_en", "retrieved_id": "84356d8d1683f6149b8e_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE EXTRACT(MONTH FROM CAST(start_date AS DATE)) = 7;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "e8cfdfba296bc283222a_noisy_en", "retrieved_id": "beff4ae8de5af55f5f09_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE EXTRACT(MONTH FROM CAST(start_date AS DATE)) = 7;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "e8cfdfba296bc283222a_hi", "retrieved_id": "beff4ae8de5af55f5f09_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE EXTRACT(MONTH FROM CAST(start_date AS DATE)) = 7;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "e8cfdfba296bc283222a_hinglish", "retrieved_id": "ee3c888ba674b0f5b845_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE EXTRACT(MONTH FROM CAST(start_date AS DATE)) = 7;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "4745f53221e07d4f5ae1_en", "retrieved_id": "75a9842978cd9cf849de_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE LOWER(display_name) LIKE LOWER('%Pune%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4745f53221e07d4f5ae1_noisy_en", "retrieved_id": "279c576638ef45226927_concrete_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE LOWER(display_name) LIKE LOWER('%Books%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "4745f53221e07d4f5ae1_hi", "retrieved_id": "7757b57351fd2c199855_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE LOWER(display_name) LIKE LOWER('%Pune%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4745f53221e07d4f5ae1_hinglish", "retrieved_id": "8084fc579ebd299cff5f_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE LOWER(display_name) LIKE LOWER('%Pune%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "fd9cafbb7e6e0ce69d06_en", "retrieved_id": "4350cfb9872b973f6709_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "fd9cafbb7e6e0ce69d06_noisy_en", "retrieved_id": "4350cfb9872b973f6709_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "fd9cafbb7e6e0ce69d06_hi", "retrieved_id": "80487a866bf00246a506_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "fd9cafbb7e6e0ce69d06_hinglish", "retrieved_id": "97bd4d1472a75cd5d314_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e02df083b1ff1b7fbcf8_en", "retrieved_id": "875583a1b3d6c73b22d6_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT exhibits.customer_name FROM exhibits JOIN exhibits_groups ON exhibits.group_id = exhibits_groups.id WHERE exhibits_groups.label = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e02df083b1ff1b7fbcf8_noisy_en", "retrieved_id": "725355658d2c31985feb_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT exhibits.customer_name FROM exhibits JOIN exhibits_groups ON exhibits.group_id = exhibits_groups.id WHERE exhibits_groups.label = 'Mumbai';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "e02df083b1ff1b7fbcf8_hi", "retrieved_id": "1f64dd4a8cd091acb5ed_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT exhibits.customer_name FROM exhibits JOIN exhibits_groups ON exhibits.group_id = exhibits_groups.id WHERE exhibits_groups.label = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e02df083b1ff1b7fbcf8_hinglish", "retrieved_id": "d946e0d878b01a80d48a_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT exhibits.customer_name FROM exhibits JOIN exhibits_groups ON exhibits.group_id = exhibits_groups.id WHERE exhibits_groups.label = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "940596942f7c5e4bf758_en", "retrieved_id": "bd6b78905a3ad03e04b6_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specimens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "940596942f7c5e4bf758_noisy_en", "retrieved_id": "bd6b78905a3ad03e04b6_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specimens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "940596942f7c5e4bf758_hi", "retrieved_id": "944772e48c9b5194fe1e_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specimens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "940596942f7c5e4bf758_hinglish", "retrieved_id": "5165d04cc5f4626621a5_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"],\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "2ed720de1d1005daa102_en", "retrieved_id": "863d3ef71bbbeba4919e_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "2ed720de1d1005daa102_noisy_en", "retrieved_id": "632bfa2e24875823150a_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "2ed720de1d1005daa102_hi", "retrieved_id": "286869979f46e9bbc61c_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "2ed720de1d1005daa102_hinglish", "retrieved_id": "4253da29b10c8d2d921b_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"books\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "8234746878d4297cb07a_en", "retrieved_id": "fb890425dce9de189858_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "8234746878d4297cb07a_noisy_en", "retrieved_id": "25ac9041dfe1446dd870_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "8234746878d4297cb07a_hi", "retrieved_id": "427fd58d6660abf6f38a_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "8234746878d4297cb07a_hinglish", "retrieved_id": "25ac9041dfe1446dd870_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "797213190eddac109b96_en", "retrieved_id": "ff73314a0982d6281641_noisy_en", "output": "{\"action\":\"call\",\"name\":\"database.describe_table\",\"arguments\":{\"table\":\"books\"}}", "metrics": {"json_valid": true, "schema_valid": false, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false, "error": "Unknown tool"}} +{"id": "797213190eddac109b96_noisy_en", "retrieved_id": "03727992a863b145c565_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "797213190eddac109b96_hi", "retrieved_id": "26ee7e71ccaee33a6162_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "797213190eddac109b96_hinglish", "retrieved_id": "0af23c07619fdf39d636_hinglish", "output": "{\"action\":\"call\",\"name\":\"database.describe_table\",\"arguments\":{\"table\":\"books\"}}", "metrics": {"json_valid": true, "schema_valid": false, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false, "error": "Unknown tool"}} +{"id": "8ebc9e868ee558a8ddfe_en", "retrieved_id": "a5928e9d2a9de389c96e_noisy_en", "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "8ebc9e868ee558a8ddfe_noisy_en", "retrieved_id": "7b86419573eea700d641_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": false, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false}} +{"id": "8ebc9e868ee558a8ddfe_hi", "retrieved_id": "d8fbf98bcb77ca92a390_hi", "output": "{\"action\":\"clarify\",\"question\":\"सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "8ebc9e868ee558a8ddfe_hinglish", "retrieved_id": "981c50947db171d5fec5_hinglish", "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "523a83d6ca07c1a1334a_en", "retrieved_id": "22a9c838578d2feeaf02_noisy_en", "output": "{\"action\":\"clarify\",\"question\":\"Which value should department equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "523a83d6ca07c1a1334a_noisy_en", "retrieved_id": "22a9c838578d2feeaf02_noisy_en", "output": "{\"action\":\"clarify\",\"question\":\"Which value should department equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "523a83d6ca07c1a1334a_hi", "retrieved_id": "677a222f59b0e9310e4e_hi", "output": "{\"action\":\"clarify\",\"question\":\"department का कौन सा मान चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "523a83d6ca07c1a1334a_hinglish", "retrieved_id": "bbdeaa042d51e3f80dcc_hinglish", "output": "{\"action\":\"clarify\",\"question\":\"department ki kaunsi value chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "2ca6667f987742cde1e7_en", "retrieved_id": "8fde1669f8ae856382b5_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": false, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false}} +{"id": "2ca6667f987742cde1e7_noisy_en", "retrieved_id": "f6d1979b8e8273fb994b_en", "output": "{\"action\":\"clarify\",\"question\":\"This connection is read-only. Would you like a SELECT query instead?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "2ca6667f987742cde1e7_hi", "retrieved_id": "a4f18f2163d8129dfc3c_hi", "output": "{\"action\":\"clarify\",\"question\":\"यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "2ca6667f987742cde1e7_hinglish", "retrieved_id": "6cad228416bb08b460f6_hinglish", "output": "{\"action\":\"clarify\",\"question\":\"Yeh connection read-only hai. Kya SELECT query chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "88b478ce80eed61bdfb4_en", "retrieved_id": "03a2503aa4caeb5b0396_en", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "88b478ce80eed61bdfb4_noisy_en", "retrieved_id": "fd5016d36c195ab50c95_noisy_en", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "88b478ce80eed61bdfb4_hi", "retrieved_id": "fcc2c16620594edcdeae_hi", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "88b478ce80eed61bdfb4_hinglish", "retrieved_id": "c6e7cd6d43a8ae36b611_hinglish", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "531ed6009fed93af93ef_en", "retrieved_id": "a927da4da3fb93076b11_en", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"indexes\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "531ed6009fed93af93ef_noisy_en", "retrieved_id": "b70d3ac89725558f5112_en", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"indexes\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "531ed6009fed93af93ef_hi", "retrieved_id": "b9a03d467eb83da89c0a_hi", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"indexes\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "531ed6009fed93af93ef_hinglish", "retrieved_id": "9a27f495c05e46011062_hinglish", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"indexes\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "13d062ce5950bde16f8f_en", "retrieved_id": "ffd875645c8eba51efea_en", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/docs/README.md\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "13d062ce5950bde16f8f_noisy_en", "retrieved_id": "ffd875645c8eba51efea_en", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/docs/README.md\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "13d062ce5950bde16f8f_hi", "retrieved_id": "80eb2f13a8c1c16513db_hi", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/docs/README.md\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "13d062ce5950bde16f8f_hinglish", "retrieved_id": "10933bf17f733deb06f6_hinglish", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/docs/README.md\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "3dbbf63e641d9abf49db_en", "retrieved_id": "7433354d41a6c516f4b6_en", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-8757\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "3dbbf63e641d9abf49db_noisy_en", "retrieved_id": "4c422d229cf597a3acdd_noisy_en_curriculum_99606", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-117\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "3dbbf63e641d9abf49db_hi", "retrieved_id": "b0c31fd3cb1afd956208_hi", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-9402\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "3dbbf63e641d9abf49db_hinglish", "retrieved_id": "6ea26747bfbf3b1aa5e0_hinglish", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-8757\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "2cf634192448e68935c1_en", "retrieved_id": "55ffca30eec9ef3fad75_noisy_en", "output": "{\"action\":\"answer\",\"text\":\"The tool returned 15 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false, "text_exact": false}} +{"id": "2cf634192448e68935c1_noisy_en", "retrieved_id": "5c43eadbe4170321c4eb_noisy_en", "output": "{\"action\":\"answer\",\"text\":\"The tool returned 5 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "2cf634192448e68935c1_hi", "retrieved_id": "c53f5a512e7d610db01c_hi", "output": "{\"action\":\"answer\",\"text\":\"टूल ने 1 पंक्तियाँ लौटाईं।\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false, "text_exact": false}} +{"id": "2cf634192448e68935c1_hinglish", "retrieved_id": "e029eeba4f40f3109a40_hinglish", "output": "{\"action\":\"answer\",\"text\":\"Tool ne 2 rows return ki.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false, "text_exact": false}} +{"id": "db7eb47a2bad512e837d_en", "retrieved_id": "130f7bdeb1d62731f52f_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "db7eb47a2bad512e837d_noisy_en", "retrieved_id": "88a8a9447d5602b34319_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "db7eb47a2bad512e837d_hi", "retrieved_id": "02501bf0d6866d9d8bf4_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "db7eb47a2bad512e837d_hinglish", "retrieved_id": "484e67e978b01025c01e_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2ffce857234ca4c32ec9_en", "retrieved_id": "427c52f3cda588a8dfa5_concrete_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT DISTINCT city FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "2ffce857234ca4c32ec9_noisy_en", "retrieved_id": "d5d019f5b163b87012e9_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT title FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2ffce857234ca4c32ec9_hi", "retrieved_id": "f6b49cc54fbec96c923e_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT title FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2ffce857234ca4c32ec9_hinglish", "retrieved_id": "b9662f007006fa757a29_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT title FROM reservations;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "41762bd116b975652e00_en", "retrieved_id": "26e86f72362cf11a9cee_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM reservations WHERE status = 'South';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "41762bd116b975652e00_noisy_en", "retrieved_id": "76f759ba99a24585f312_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM reservations WHERE status = 'South';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "41762bd116b975652e00_hi", "retrieved_id": "454de75d753988450bb8_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE status = 'South';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "41762bd116b975652e00_hinglish", "retrieved_id": "454de75d753988450bb8_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE status = 'South';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "358364da33322bbf17e2_en", "retrieved_id": "3464e3db347b8262b70a_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT title FROM exhibits WHERE city = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "358364da33322bbf17e2_noisy_en", "retrieved_id": "b317976d8eb8c32ee7ab_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT title FROM exhibits WHERE city = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "358364da33322bbf17e2_hi", "retrieved_id": "b0afc3e3af9d4dc33f21_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT title FROM exhibits WHERE city = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "358364da33322bbf17e2_hinglish", "retrieved_id": "5d64681ab13e5ac23030_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT title FROM exhibits WHERE city = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2f545db764e4c6cd353d_en", "retrieved_id": "b2d5adea6ae385989d97_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE salary > 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2f545db764e4c6cd353d_noisy_en", "retrieved_id": "ee831366d69fd0cbaff0_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE salary > 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2f545db764e4c6cd353d_hi", "retrieved_id": "93630964314d2a1070ff_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE description IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "2f545db764e4c6cd353d_hinglish", "retrieved_id": "b2d5adea6ae385989d97_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE salary > 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "aa867c871299c26d5402_en", "retrieved_id": "0cdd37ad69e76d5f26cc_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE rating < 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "aa867c871299c26d5402_noisy_en", "retrieved_id": "41d106714a599beb627d_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE rating < 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "aa867c871299c26d5402_hi", "retrieved_id": "5a6fa15cf2837ad23736_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE rating < 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "aa867c871299c26d5402_hinglish", "retrieved_id": "0cdd37ad69e76d5f26cc_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE rating < 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "3fdbfd3a4d0df901f143_en", "retrieved_id": "90607a080fdcde834c8d_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE amount > 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "3fdbfd3a4d0df901f143_noisy_en", "retrieved_id": "5a1b627a27ad05c33f30_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE amount < 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "3fdbfd3a4d0df901f143_hi", "retrieved_id": "d75197e3cbaf37ffab4c_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE amount >= 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "3fdbfd3a4d0df901f143_hinglish", "retrieved_id": "103a21087bbabce1d29a_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE amount >= 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "37a16740c32af37d3b15_en", "retrieved_id": "734a4347b6aef6b54122_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE score > 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "37a16740c32af37d3b15_noisy_en", "retrieved_id": "b7afebc36043fd08cce2_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE score <= 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "37a16740c32af37d3b15_hi", "retrieved_id": "c73f27d82996099d371b_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE score >= 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "37a16740c32af37d3b15_hinglish", "retrieved_id": "99f9c207c7acb4e5dfe7_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE score <= 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ccbb8ac5e43114f43047_en", "retrieved_id": "ce663569111e9035dd99_concrete_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE balance BETWEEN 50 AND 60;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "ccbb8ac5e43114f43047_noisy_en", "retrieved_id": "eb21df901f88079702af_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE balance BETWEEN 50 AND 100;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ccbb8ac5e43114f43047_hi", "retrieved_id": "80a601332d98498fd1a8_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE balance BETWEEN 50 AND 100;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ccbb8ac5e43114f43047_hinglish", "retrieved_id": "eb21df901f88079702af_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE balance BETWEEN 50 AND 100;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8643d909ff9369a5491c_en", "retrieved_id": "924cf25da228291be3d0_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE kind = 'Mumbai' AND rating > 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8643d909ff9369a5491c_noisy_en", "retrieved_id": "924cf25da228291be3d0_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE kind = 'Mumbai' AND rating > 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8643d909ff9369a5491c_hi", "retrieved_id": "6b9ae5e3d5c6b82e81b0_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE rating > 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "8643d909ff9369a5491c_hinglish", "retrieved_id": "c304edcc689fd9487769_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE kind = 'Mumbai' AND rating > 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "a7afc1ab21c25c700603_en", "retrieved_id": "d65f0b5705912722a19c_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE status IN ('Electronics', 'Electronics');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "a7afc1ab21c25c700603_noisy_en", "retrieved_id": "d65f0b5705912722a19c_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE status IN ('Electronics', 'Electronics');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "a7afc1ab21c25c700603_hi", "retrieved_id": "31917a12777c21c921b7_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE status IN ('Mumbai', 'Electronics');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a7afc1ab21c25c700603_hinglish", "retrieved_id": "8f3dddcc62bbdd7ab668_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE status IN ('North', 'Electronics');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "8fa63f2c53b1bc742d49_en", "retrieved_id": "bda2d640bb8440ba316d_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE title IS NULL;\",\"project_id\":\"demo_inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8fa63f2c53b1bc742d49_noisy_en", "retrieved_id": "2979500f0ecfcf0ceeaf_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE title IS NULL;\",\"project_id\":\"demo_inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8fa63f2c53b1bc742d49_hi", "retrieved_id": "6f58fb9ba5d390237de3_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE NOT title IS NULL;\",\"project_id\":\"demo_inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "8fa63f2c53b1bc742d49_hinglish", "retrieved_id": "5bb801582af241e0e0d2_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE NOT title IS NULL;\",\"project_id\":\"demo_inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "0c15b441e3e3ee8bb0df_en", "retrieved_id": "d445a01cd3d55cf35b81_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0c15b441e3e3ee8bb0df_noisy_en", "retrieved_id": "6458a3fe5e6684bb7028_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "0c15b441e3e3ee8bb0df_hi", "retrieved_id": "739f12ded833785d6962_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE NOT name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "0c15b441e3e3ee8bb0df_hinglish", "retrieved_id": "1971a91ea9f92ae7bf58_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE NOT name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "bf7bc588f62f075ec1ca_en", "retrieved_id": "c40602913c502e7ccda9_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT COUNT(*) FROM exhibits WHERE kind = 'Mumbai';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "bf7bc588f62f075ec1ca_noisy_en", "retrieved_id": "6040c6883947ec5b3821_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM exhibits;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bf7bc588f62f075ec1ca_hi", "retrieved_id": "2d416c705002edc7a1af_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM exhibits;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bf7bc588f62f075ec1ca_hinglish", "retrieved_id": "33e0077bee491c810f9d_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM exhibits;\",\"project_id\":\"demo_exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7d18561ec942f18f1299_en", "retrieved_id": "21ee14be5ee9ce8ba3e5_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM reservations WHERE status = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7d18561ec942f18f1299_noisy_en", "retrieved_id": "eee0803de05a20518856_concrete_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM reservations WHERE status = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7d18561ec942f18f1299_hi", "retrieved_id": "5abf1c647897ccd8a567_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE status IN ('North', 'O''Reilly');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "7d18561ec942f18f1299_hinglish", "retrieved_id": "21ee14be5ee9ce8ba3e5_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM reservations WHERE status = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "28530a7607feca135b73_en", "retrieved_id": "e4a4498900568bd25b0f_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT kind, SUM(price) FROM specimens GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "28530a7607feca135b73_noisy_en", "retrieved_id": "a33bc62f70795ecfa8a2_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(price) FROM specimens;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "28530a7607feca135b73_hi", "retrieved_id": "3c25f8671f95114b6257_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(price) FROM specimens;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "28530a7607feca135b73_hinglish", "retrieved_id": "caba0067fdd24c562152_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT SUM(price) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ad0842eb95739a0c88ec_en", "retrieved_id": "a1369f19e8dcd07436c9_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ad0842eb95739a0c88ec_noisy_en", "retrieved_id": "e16c4dae0f3afac7f310_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ad0842eb95739a0c88ec_hi", "retrieved_id": "52abab06839657236449_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ad0842eb95739a0c88ec_hinglish", "retrieved_id": "3442026d043514bd5f1c_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c466f263014e2eb31cf5_en", "retrieved_id": "f799074a7799eb92b1c0_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT MAX(rating) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c466f263014e2eb31cf5_noisy_en", "retrieved_id": "ad5c6763cee83fde6df0_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT MAX(rating) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c466f263014e2eb31cf5_hi", "retrieved_id": "2bd4dee285ff24521ff6_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT SUM(rating) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "c466f263014e2eb31cf5_hinglish", "retrieved_id": "29e87bc0175bf44f2332_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT MAX(rating) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1395b37e5ba28fa3c134_en", "retrieved_id": "4d035a536476fdd62299_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(amount) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1395b37e5ba28fa3c134_noisy_en", "retrieved_id": "2191ec0e7a2e2e4e18b2_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(amount) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1395b37e5ba28fa3c134_hi", "retrieved_id": "60337122c76970ce6f93_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(amount) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1395b37e5ba28fa3c134_hinglish", "retrieved_id": "9a6db088671e3d6461b8_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(amount) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e114d1f216088c4302b2_en", "retrieved_id": "f4d0cc8f44c31aba25fd_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT DISTINCT department FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e114d1f216088c4302b2_noisy_en", "retrieved_id": "f4d0cc8f44c31aba25fd_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT DISTINCT department FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e114d1f216088c4302b2_hi", "retrieved_id": "f4d0cc8f44c31aba25fd_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT DISTINCT department FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e114d1f216088c4302b2_hinglish", "retrieved_id": "f0e8a1aa17ebe2943f9a_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE customer_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "71349134f8ae16809501_en", "retrieved_id": "de723244833eb9acd9b0_concrete_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "71349134f8ae16809501_noisy_en", "retrieved_id": "dbbc87df17789dcf16c7_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC NULLS LAST;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "71349134f8ae16809501_hi", "retrieved_id": "5f714fc20f63b02a2389_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC NULLS LAST;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "71349134f8ae16809501_hinglish", "retrieved_id": "9360abd24d181d80b919_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC NULLS LAST;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "b10e3bcce6871b79df41_en", "retrieved_id": "36010b0cd2ebc005b766_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY CASE WHEN price IS NULL THEN 1 ELSE 0 END DESC, price DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "b10e3bcce6871b79df41_noisy_en", "retrieved_id": "36010b0cd2ebc005b766_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY CASE WHEN price IS NULL THEN 1 ELSE 0 END DESC, price DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "b10e3bcce6871b79df41_hi", "retrieved_id": "6a30e3aab87d8b8fac24_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY CASE WHEN price IS NULL THEN 1 ELSE 0 END, price ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "b10e3bcce6871b79df41_hinglish", "retrieved_id": "d49736acab21961ee1c4_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY price ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4c5f91f7793bd0200f31_en", "retrieved_id": "eb4e726353b860e7f2cb_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY CASE WHEN amount IS NULL THEN 1 ELSE 0 END DESC, amount DESC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "4c5f91f7793bd0200f31_noisy_en", "retrieved_id": "a660459963aa550973d6_concrete_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY amount ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "4c5f91f7793bd0200f31_hi", "retrieved_id": "0bf2604249c599c4c112_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY amount DESC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4c5f91f7793bd0200f31_hinglish", "retrieved_id": "1982f2d4c79276d08d58_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY amount DESC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bfb3f8261ddb2fc2bf28_en", "retrieved_id": "44050a272a88bea75138_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens ORDER BY price ASC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bfb3f8261ddb2fc2bf28_noisy_en", "retrieved_id": "44050a272a88bea75138_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens ORDER BY price ASC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bfb3f8261ddb2fc2bf28_hi", "retrieved_id": "44050a272a88bea75138_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens ORDER BY price ASC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bfb3f8261ddb2fc2bf28_hinglish", "retrieved_id": "44050a272a88bea75138_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens ORDER BY price ASC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4dac1dcf2ef14bfee2d2_en", "retrieved_id": "39ffd26f283fc2242233_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4dac1dcf2ef14bfee2d2_noisy_en", "retrieved_id": "6ad175c1da38937a1c73_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4dac1dcf2ef14bfee2d2_hi", "retrieved_id": "11be592bce7c6de29c95_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4dac1dcf2ef14bfee2d2_hinglish", "retrieved_id": "c2e6ba167b989ceb36a1_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "46b2f9dcb5c441ef3e8e_en", "retrieved_id": "37741990cedfc85c3b87_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT kind, SUM(rating) FROM exhibits GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "46b2f9dcb5c441ef3e8e_noisy_en", "retrieved_id": "dd2fb2aacc30b9922bcb_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT kind, SUM(rating) FROM exhibits GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "46b2f9dcb5c441ef3e8e_hi", "retrieved_id": "37741990cedfc85c3b87_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT kind, SUM(rating) FROM exhibits GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "46b2f9dcb5c441ef3e8e_hinglish", "retrieved_id": "92e9f6f1139932d803a6_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT kind, SUM(rating) FROM exhibits GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bd5fbea0dbbb3414c53c_en", "retrieved_id": "97445a75f67055563e4b_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 10;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "bd5fbea0dbbb3414c53c_noisy_en", "retrieved_id": "efbc7f4492f9398e5970_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 0;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bd5fbea0dbbb3414c53c_hi", "retrieved_id": "bed6bb82a599f4f59f4b_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 0;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bd5fbea0dbbb3414c53c_hinglish", "retrieved_id": "d592e70e8a3eac30079e_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 0;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "30a7ec8a99d4ee6b6d0e_en", "retrieved_id": "4afe0855bc7cab00ef70_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE order_date >= '2026-12-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "30a7ec8a99d4ee6b6d0e_noisy_en", "retrieved_id": "b094cf1912edbb738f6f_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE order_date >= '2026-12-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "30a7ec8a99d4ee6b6d0e_hi", "retrieved_id": "4afe0855bc7cab00ef70_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE order_date >= '2026-12-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "30a7ec8a99d4ee6b6d0e_hinglish", "retrieved_id": "5577426648e5588099f5_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE order_date >= '2026-11-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "eab95ae98810f2228ba6_en", "retrieved_id": "54d6b19691b0ae29f4f4_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE YEAR(start_date) = 2026;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "eab95ae98810f2228ba6_noisy_en", "retrieved_id": "d7f17f4e1fbc730b50f0_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE EXTRACT(YEAR FROM start_date) = 2026;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "eab95ae98810f2228ba6_hi", "retrieved_id": "d7f17f4e1fbc730b50f0_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE EXTRACT(YEAR FROM start_date) = 2026;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "eab95ae98810f2228ba6_hinglish", "retrieved_id": "5829e621fdb043061ee3_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE YEAR(start_date) = 2026;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "24661fe309a7c9930145_en", "retrieved_id": "e4e3d62e41cf067f7dce_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE MONTH(recorded_at) = 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "24661fe309a7c9930145_noisy_en", "retrieved_id": "9d2b356f5ff451f7b6ed_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE MONTH(recorded_at) = 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "24661fe309a7c9930145_hi", "retrieved_id": "6e9bb556a5963cdf5030_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE MONTH(recorded_at) = 12;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "24661fe309a7c9930145_hinglish", "retrieved_id": "552f56c716edc8eb10d5_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE MONTH(recorded_at) = 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "29bdf95ec36d6924a576_en", "retrieved_id": "a0d363ff3f1d68a099bd_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE LOWER(item_name) LIKE LOWER('%O''Reilly%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "29bdf95ec36d6924a576_noisy_en", "retrieved_id": "279c576638ef45226927_concrete_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE LOWER(item_name) LIKE LOWER('%Books%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "29bdf95ec36d6924a576_hi", "retrieved_id": "e8f36218d6da0f2cdffd_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE LOWER(item_name) LIKE LOWER('%O''Reilly%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "29bdf95ec36d6924a576_hinglish", "retrieved_id": "8a0c25e1d4f47a787a27_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE LOWER(item_name) LIKE LOWER('%O''Reilly%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "53220f60789b4f5e4046_en", "retrieved_id": "22701200b58e29b4c0ab_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "53220f60789b4f5e4046_noisy_en", "retrieved_id": "22701200b58e29b4c0ab_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "53220f60789b4f5e4046_hi", "retrieved_id": "949b6e0ae2bd47e7b2a2_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "53220f60789b4f5e4046_hinglish", "retrieved_id": "5f940b0e9d7dfa863d6b_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1b32b024d0c001fd9ae4_en", "retrieved_id": "a67c49bd271104e6d81b_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1b32b024d0c001fd9ae4_noisy_en", "retrieved_id": "c207894e41bd7b9014aa_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1b32b024d0c001fd9ae4_hi", "retrieved_id": "61ae49349949dc399a63_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1b32b024d0c001fd9ae4_hinglish", "retrieved_id": "e60aa1d2eef307b6933f_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9bf8948becf74e81a103_en", "retrieved_id": "bd6b78905a3ad03e04b6_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "9bf8948becf74e81a103_noisy_en", "retrieved_id": "bd6b78905a3ad03e04b6_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "9bf8948becf74e81a103_hi", "retrieved_id": "944772e48c9b5194fe1e_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "9bf8948becf74e81a103_hinglish", "retrieved_id": "5165d04cc5f4626621a5_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "e4a81e1122226c7b26d9_en", "retrieved_id": "863d3ef71bbbeba4919e_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "e4a81e1122226c7b26d9_noisy_en", "retrieved_id": "632bfa2e24875823150a_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "e4a81e1122226c7b26d9_hi", "retrieved_id": "286869979f46e9bbc61c_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "e4a81e1122226c7b26d9_hinglish", "retrieved_id": "4253da29b10c8d2d921b_hinglish", "output": "{\"action\":\"call\",\"name\":\"describe_table\",\"arguments\":{\"table\":\"books\"}}", "metrics": {"json_valid": true, "schema_valid": false, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false, "error": "Unknown tool"}} +{"id": "41f5fbcefb9d550068a0_en", "retrieved_id": "fb890425dce9de189858_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specimens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "41f5fbcefb9d550068a0_noisy_en", "retrieved_id": "25ac9041dfe1446dd870_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"],\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "41f5fbcefb9d550068a0_hi", "retrieved_id": "427fd58d6660abf6f38a_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"],\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "41f5fbcefb9d550068a0_hinglish", "retrieved_id": "25ac9041dfe1446dd870_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"],\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "6f162db917ded36d0df0_en", "retrieved_id": "ff73314a0982d6281641_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"books\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "6f162db917ded36d0df0_noisy_en", "retrieved_id": "03727992a863b145c565_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "6f162db917ded36d0df0_hi", "retrieved_id": "26ee7e71ccaee33a6162_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "6f162db917ded36d0df0_hinglish", "retrieved_id": "0af23c07619fdf39d636_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"books\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "6b9bacf4d9aa380af183_en", "retrieved_id": "a5928e9d2a9de389c96e_noisy_en", "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "6b9bacf4d9aa380af183_noisy_en", "retrieved_id": "7b86419573eea700d641_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": false, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false}} +{"id": "6b9bacf4d9aa380af183_hi", "retrieved_id": "d8fbf98bcb77ca92a390_hi", "output": "{\"action\":\"clarify\",\"question\":\"सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "6b9bacf4d9aa380af183_hinglish", "retrieved_id": "981c50947db171d5fec5_hinglish", "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "5d05a2e11d5abab8423e_en", "retrieved_id": "a44004f0eb7cc5872af0_noisy_en", "output": "{\"action\":\"clarify\",\"question\":\"Which value should region equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "5d05a2e11d5abab8423e_noisy_en", "retrieved_id": "a44004f0eb7cc5872af0_noisy_en", "output": "{\"action\":\"clarify\",\"question\":\"Which value should region equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "5d05a2e11d5abab8423e_hi", "retrieved_id": "90ab6a4e7d8fa9ce50fb_hi", "output": "{\"action\":\"clarify\",\"question\":\"region का कौन सा मान चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "5d05a2e11d5abab8423e_hinglish", "retrieved_id": "8c963f079269e7395aa1_hinglish", "output": "{\"action\":\"clarify\",\"question\":\"region ki kaunsi value chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "65666802d11552ba1fc2_en", "retrieved_id": "8fde1669f8ae856382b5_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": false, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false}} +{"id": "65666802d11552ba1fc2_noisy_en", "retrieved_id": "f6d1979b8e8273fb994b_en", "output": "{\"action\":\"clarify\",\"question\":\"This connection is read-only. Would you like a SELECT query instead?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "65666802d11552ba1fc2_hi", "retrieved_id": "a4f18f2163d8129dfc3c_hi", "output": "{\"action\":\"clarify\",\"question\":\"यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "65666802d11552ba1fc2_hinglish", "retrieved_id": "6cad228416bb08b460f6_hinglish", "output": "{\"action\":\"clarify\",\"question\":\"Yeh connection read-only hai. Kya SELECT query chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "6ba167225d0a0648e865_en", "retrieved_id": "60c088bfa4c1573171ce_en", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Lucknow\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "6ba167225d0a0648e865_noisy_en", "retrieved_id": "19be7f84e5c1fc8bab22_noisy_en", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Lucknow\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "6ba167225d0a0648e865_hi", "retrieved_id": "19be7f84e5c1fc8bab22_hi", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Lucknow\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "6ba167225d0a0648e865_hinglish", "retrieved_id": "c8617cfd9655faabcf9e_hinglish", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Lucknow\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "d84c7c968e50589a3321_en", "retrieved_id": "1b5afb342f9879ce6e08_en", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"API authentication\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "d84c7c968e50589a3321_noisy_en", "retrieved_id": "e8457f39588335416df5_en", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"API authentication\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "d84c7c968e50589a3321_hi", "retrieved_id": "f12b3b98df01a6987872_hi", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"API authentication\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "d84c7c968e50589a3321_hinglish", "retrieved_id": "335e9ac94d47eb84a1e6_hinglish", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"API authentication\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "ae382a2348421f540f9e_en", "retrieved_id": "3662e94e29edbee93120_en", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "ae382a2348421f540f9e_noisy_en", "retrieved_id": "3662e94e29edbee93120_en", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "ae382a2348421f540f9e_hi", "retrieved_id": "28f97a3353e78f589cfe_hi", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "ae382a2348421f540f9e_hinglish", "retrieved_id": "0e78e9ded20fe5272477_hinglish", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "266ea8aafdf18d7c9212_en", "retrieved_id": "7433354d41a6c516f4b6_en", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-8757\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "266ea8aafdf18d7c9212_noisy_en", "retrieved_id": "4c422d229cf597a3acdd_noisy_en_curriculum_99606", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-117\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "266ea8aafdf18d7c9212_hi", "retrieved_id": "b0c31fd3cb1afd956208_hi", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-9402\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "266ea8aafdf18d7c9212_hinglish", "retrieved_id": "6ea26747bfbf3b1aa5e0_hinglish", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-8757\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "e9cfe7f6c91a2434d315_en", "retrieved_id": "56626f78a1718e142007_en", "output": "{\"action\":\"answer\",\"text\":\"The tool returned 102 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "e9cfe7f6c91a2434d315_noisy_en", "retrieved_id": "d2b3293aaabb219b6010_noisy_en", "output": "{\"action\":\"answer\",\"text\":\"The tool returned 102 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "e9cfe7f6c91a2434d315_hi", "retrieved_id": "90e49bed3d5c138efc4b_hi", "output": "{\"action\":\"answer\",\"text\":\"टूल ने 102 पंक्तियाँ लौटाईं।\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "e9cfe7f6c91a2434d315_hinglish", "retrieved_id": "e029eeba4f40f3109a40_hinglish", "output": "{\"action\":\"answer\",\"text\":\"Tool ne 2 rows return ki.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false, "text_exact": false}} +{"id": "7a843b154fbd48aac50b_en", "retrieved_id": "130f7bdeb1d62731f52f_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7a843b154fbd48aac50b_noisy_en", "retrieved_id": "88a8a9447d5602b34319_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7a843b154fbd48aac50b_hi", "retrieved_id": "02501bf0d6866d9d8bf4_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7a843b154fbd48aac50b_hinglish", "retrieved_id": "484e67e978b01025c01e_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e6380b8531b4b3eb7e38_en", "retrieved_id": "427c52f3cda588a8dfa5_concrete_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT DISTINCT category FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "e6380b8531b4b3eb7e38_noisy_en", "retrieved_id": "b2757aefcc8bc0a7549e_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e6380b8531b4b3eb7e38_hi", "retrieved_id": "3aac2f1448f136869091_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e6380b8531b4b3eb7e38_hinglish", "retrieved_id": "436de5794ee935ad91ac_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "19d562dac8a4125562f6_en", "retrieved_id": "c240f9f94886fddc3cba_concrete_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE city = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "19d562dac8a4125562f6_noisy_en", "retrieved_id": "fbe3ca6a45a59c91d568_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM inspections WHERE city = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "19d562dac8a4125562f6_hi", "retrieved_id": "4b4dc865bbc2f957a5ef_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM inspections WHERE city = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "19d562dac8a4125562f6_hinglish", "retrieved_id": "7ba3e80650c610585052_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE city = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d0346c74f1cfe556b2cb_en", "retrieved_id": "fcc2a0204203d73148e3_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT description FROM inspections WHERE status = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d0346c74f1cfe556b2cb_noisy_en", "retrieved_id": "cf17c45517d6b2205633_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT description FROM inspections WHERE status = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d0346c74f1cfe556b2cb_hi", "retrieved_id": "aa12aa7a83c4a4e070aa_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT description FROM inspections WHERE status = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d0346c74f1cfe556b2cb_hinglish", "retrieved_id": "94524d174daf1430782e_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT description FROM inspections WHERE status = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6519cd640e9e4ab22a6e_en", "retrieved_id": "330001a7e7402edb7b9a_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE score <= 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "6519cd640e9e4ab22a6e_noisy_en", "retrieved_id": "b29589b85da26af925a2_concrete_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE score > 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "6519cd640e9e4ab22a6e_hi", "retrieved_id": "93630964314d2a1070ff_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE customer_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "6519cd640e9e4ab22a6e_hinglish", "retrieved_id": "4cdbd72a8fb19b994edf_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE score < 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "56189c5cdc014dd771be_en", "retrieved_id": "9a55d54e98ef6eec8304_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE balance < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "56189c5cdc014dd771be_noisy_en", "retrieved_id": "0d87e4597b8030db1d39_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE balance < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "56189c5cdc014dd771be_hi", "retrieved_id": "6c50e81ff0658a9ae684_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE balance < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "56189c5cdc014dd771be_hinglish", "retrieved_id": "807599fab71788aa599e_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE balance < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5c6734bdfead38b51752_en", "retrieved_id": "d7395c1b839773e90e4a_concrete_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE quantity <= 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "5c6734bdfead38b51752_noisy_en", "retrieved_id": "044ca211c7658be405e7_noisy_en_curriculum_99606", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE quantity >= 10000;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5c6734bdfead38b51752_hi", "retrieved_id": "af6e41a958ceb97494ac_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE quantity >= 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5c6734bdfead38b51752_hinglish", "retrieved_id": "b5f87dae75d5de42adf3_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE quantity >= 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d58041d2866c015bc923_en", "retrieved_id": "944f318073a2bcd9624d_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE amount > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "d58041d2866c015bc923_noisy_en", "retrieved_id": "effd7733bf11c5777e47_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE amount <= 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d58041d2866c015bc923_hi", "retrieved_id": "394d070f071f9f834849_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE amount >= 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "d58041d2866c015bc923_hinglish", "retrieved_id": "273548ab688043ce968f_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE amount <= 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6dd9f03c26298630777f_en", "retrieved_id": "b2078b53152f326792d2_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE score BETWEEN 1 AND 101;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6dd9f03c26298630777f_noisy_en", "retrieved_id": "d8750c4dfe2c30ac7230_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE score BETWEEN 1 AND 101;\",\"project_id\":\"demo_inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6dd9f03c26298630777f_hi", "retrieved_id": "25d16cffc60709de4959_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE score BETWEEN 1 AND 101;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6dd9f03c26298630777f_hinglish", "retrieved_id": "25d16cffc60709de4959_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE score BETWEEN 1 AND 101;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "393d51bc9e30d68b474e_en", "retrieved_id": "515c393982a754f43063_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE cost > 5;\",\"project_id\":\"demo_inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "393d51bc9e30d68b474e_noisy_en", "retrieved_id": "3d1dd294a6f2022aa870_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE status = 'active' AND cost > 20;\",\"project_id\":\"demo_inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "393d51bc9e30d68b474e_hi", "retrieved_id": "8b036fe3469b645402c6_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE status = 'active' AND cost > 50;\",\"project_id\":\"demo_inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "393d51bc9e30d68b474e_hinglish", "retrieved_id": "38f6f8f7a2683654db95_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE status = 'active' AND cost > 20;\",\"project_id\":\"demo_inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "28cb6c4bf0d7caac91c9_en", "retrieved_id": "31917a12777c21c921b7_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'Electronics');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "28cb6c4bf0d7caac91c9_noisy_en", "retrieved_id": "31917a12777c21c921b7_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'Electronics');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "28cb6c4bf0d7caac91c9_hi", "retrieved_id": "31917a12777c21c921b7_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'Electronics');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "28cb6c4bf0d7caac91c9_hinglish", "retrieved_id": "db4ea9df05d68f0808f0_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'Electronics');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a0bc7565d9d93d546770_en", "retrieved_id": "b938809a21ca895bfdaa_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a0bc7565d9d93d546770_noisy_en", "retrieved_id": "999181602f630fe41f53_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a0bc7565d9d93d546770_hi", "retrieved_id": "f1b3784f5142d5df00b8_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE item_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "a0bc7565d9d93d546770_hinglish", "retrieved_id": "c39505a9b595f6ac6b82_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "3eea55cd4532636ddffe_en", "retrieved_id": "c7bb7b4dfc547acf0b9c_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE display_name IS NOT NULL;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "3eea55cd4532636ddffe_noisy_en", "retrieved_id": "348951272c18ab797a78_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE display_name IS NULL;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "3eea55cd4532636ddffe_hi", "retrieved_id": "c7bb7b4dfc547acf0b9c_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE display_name IS NOT NULL;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "3eea55cd4532636ddffe_hinglish", "retrieved_id": "1bbb8e0920a5610a79fb_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE display_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "32876ff450ca027d658e_en", "retrieved_id": "c40602913c502e7ccda9_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens WHERE category = 'Mumbai';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "32876ff450ca027d658e_noisy_en", "retrieved_id": "6040c6883947ec5b3821_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "32876ff450ca027d658e_hi", "retrieved_id": "2d416c705002edc7a1af_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "32876ff450ca027d658e_hinglish", "retrieved_id": "33e0077bee491c810f9d_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1523073b9b419fc60ded_en", "retrieved_id": "cbc3795c0ca21a9b8c5a_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "1523073b9b419fc60ded_noisy_en", "retrieved_id": "f237dec70fbabb886e7f_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "1523073b9b419fc60ded_hi", "retrieved_id": "ae3fd37c476f1c7921ab_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE city = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "1523073b9b419fc60ded_hinglish", "retrieved_id": "37abdc225ded84224f90_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM specimens WHERE city = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "924dbd9ad819c0a58b24_en", "retrieved_id": "60aa3677ed597c9270b0_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT status, SUM(salary) FROM inspections GROUP BY status;\",\"project_id\":\"demo_inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "924dbd9ad819c0a58b24_noisy_en", "retrieved_id": "b09ff9455bc114714c60_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(salary) FROM inspections;\",\"project_id\":\"demo_inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "924dbd9ad819c0a58b24_hi", "retrieved_id": "d1c39ccebc6f22cb6281_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(salary) FROM inspections;\",\"project_id\":\"demo_inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "924dbd9ad819c0a58b24_hinglish", "retrieved_id": "723505e6d0cbab04424f_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(salary) FROM inspections;\",\"project_id\":\"demo_inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "16fbb58e3a07a1240f50_en", "retrieved_id": "5c7028d7d36fd491bc2e_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT AVG(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "16fbb58e3a07a1240f50_noisy_en", "retrieved_id": "a441a9c205497a51a291_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT AVG(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "16fbb58e3a07a1240f50_hi", "retrieved_id": "5c7028d7d36fd491bc2e_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT AVG(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "16fbb58e3a07a1240f50_hinglish", "retrieved_id": "6bb8ee776b961884c402_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT AVG(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e91a7141af66518c19a2_en", "retrieved_id": "5ac7676141ce1a7a2b5d_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(cost) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e91a7141af66518c19a2_noisy_en", "retrieved_id": "ed835cf14e68ad586b67_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(cost) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e91a7141af66518c19a2_hi", "retrieved_id": "8166b903f1a9ad4acd2d_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(cost) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "e91a7141af66518c19a2_hinglish", "retrieved_id": "4ad4f4baf60b1a178bcb_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(cost) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "0ffff93d2f774a97e3f3_en", "retrieved_id": "d5cc3d76002f556ee9d3_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(price) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0ffff93d2f774a97e3f3_noisy_en", "retrieved_id": "da9e0a0592e6238536af_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(price) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0ffff93d2f774a97e3f3_hi", "retrieved_id": "e0e3d410c4591e668ca3_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(price) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0ffff93d2f774a97e3f3_hinglish", "retrieved_id": "bdf01655d06fc4dc9de0_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(price) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b0f56c14239ba5d0227f_en", "retrieved_id": "bffa441ff713738307bd_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT DISTINCT region FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b0f56c14239ba5d0227f_noisy_en", "retrieved_id": "bffa441ff713738307bd_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT DISTINCT region FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b0f56c14239ba5d0227f_hi", "retrieved_id": "85aa2fcfc30450a97812_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT DISTINCT region FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b0f56c14239ba5d0227f_hinglish", "retrieved_id": "f0e8a1aa17ebe2943f9a_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "f7257c4a2eded61f5c86_en", "retrieved_id": "eae5287706bec5ee520e_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY score DESC;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f7257c4a2eded61f5c86_noisy_en", "retrieved_id": "955a7d5f0d3cb038c3fa_concrete_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY score ASC NULLS FIRST;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "f7257c4a2eded61f5c86_hi", "retrieved_id": "190107647feb114e3236_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY score DESC NULLS LAST;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "f7257c4a2eded61f5c86_hinglish", "retrieved_id": "9360abd24d181d80b919_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY score DESC NULLS LAST;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "8439e708b96823b39172_en", "retrieved_id": "037483db4f53f024a398_concrete_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8439e708b96823b39172_noisy_en", "retrieved_id": "0e970032db68cbe413d7_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "8439e708b96823b39172_hi", "retrieved_id": "664c0e4ca596b55418fa_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8439e708b96823b39172_hinglish", "retrieved_id": "4e50edd9b0eb89b9f736_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9bb888f912ca90f30c9a_en", "retrieved_id": "a6540b5f2c0929241f31_concrete_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY rating ASC NULLS FIRST LIMIT 20;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "9bb888f912ca90f30c9a_noisy_en", "retrieved_id": "f0ae21259dc2404f0fe7_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY rating DESC NULLS LAST LIMIT 20;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "9bb888f912ca90f30c9a_hi", "retrieved_id": "36a2047a9b8d5e928dc2_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY rating DESC NULLS LAST LIMIT 3;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "9bb888f912ca90f30c9a_hinglish", "retrieved_id": "8dc7bed1546d4cec4da6_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY rating DESC NULLS LAST LIMIT 20;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "dd6cdefd15edcdcddb76_en", "retrieved_id": "1b8db2fed29343d4ab22_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY salary ASC NULLS FIRST LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "dd6cdefd15edcdcddb76_noisy_en", "retrieved_id": "1b8db2fed29343d4ab22_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY salary ASC NULLS FIRST LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "dd6cdefd15edcdcddb76_hi", "retrieved_id": "1b8db2fed29343d4ab22_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY salary ASC NULLS FIRST LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "dd6cdefd15edcdcddb76_hinglish", "retrieved_id": "a326401a7479b752fb44_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY salary ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f4fb7fafb95b5c8287f5_en", "retrieved_id": "2fe2748b6dc2b31719a4_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT department, COUNT(*) FROM reservations GROUP BY department;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f4fb7fafb95b5c8287f5_noisy_en", "retrieved_id": "0ed04804842617d63993_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT department, COUNT(*) FROM reservations GROUP BY department;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f4fb7fafb95b5c8287f5_hi", "retrieved_id": "cbcd01ed06edc207269b_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT department, COUNT(*) FROM reservations GROUP BY department;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f4fb7fafb95b5c8287f5_hinglish", "retrieved_id": "7f0a16123b0e92f23bef_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT department, COUNT(*) FROM reservations GROUP BY department;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "feb75e02c031a950e223_en", "retrieved_id": "7a20839894b269ec16f4_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT category, SUM(cost) FROM inspections GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "feb75e02c031a950e223_noisy_en", "retrieved_id": "567517c0a091e1859e2e_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT category, SUM(cost) FROM inspections GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "feb75e02c031a950e223_hi", "retrieved_id": "4e94a4987bdd6f5e319b_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT category, SUM(cost) FROM inspections GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "feb75e02c031a950e223_hinglish", "retrieved_id": "d7fcdcccec8b74d574c3_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT category, SUM(cost) FROM inspections GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f97582c94f2100eb1007_en", "retrieved_id": "403ef8b11dcc84ac3142_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f97582c94f2100eb1007_noisy_en", "retrieved_id": "7544647e199d8d140e2d_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f97582c94f2100eb1007_hi", "retrieved_id": "7d84adc828af717e1745_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f97582c94f2100eb1007_hinglish", "retrieved_id": "403ef8b11dcc84ac3142_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "55c82b78d80d85926577_en", "retrieved_id": "4205a2e941b427725fab_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE start_date >= '2026-09-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "55c82b78d80d85926577_noisy_en", "retrieved_id": "4205a2e941b427725fab_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE start_date >= '2026-09-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "55c82b78d80d85926577_hi", "retrieved_id": "074597366d1789819862_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE start_date >= '2022-09-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "55c82b78d80d85926577_hinglish", "retrieved_id": "3504c2b7973f4dd08d71_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE start_date >= '2026-09-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a7efbaf306821c7ab040_en", "retrieved_id": "07b14f7c8187736bde5e_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE EXTRACT(YEAR FROM CAST(order_date AS DATE)) = 2023;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "a7efbaf306821c7ab040_noisy_en", "retrieved_id": "83a54cdf86471d7f4385_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE EXTRACT(YEAR FROM order_date) = 2023;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a7efbaf306821c7ab040_hi", "retrieved_id": "dd4b33ed1756d76ea48e_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE EXTRACT(YEAR FROM CAST(order_date AS DATE)) = 2023;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "a7efbaf306821c7ab040_hinglish", "retrieved_id": "6ad0f38bfaa5d6181467_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE EXTRACT(YEAR FROM order_date) = 2023;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7a33de7fd7159808b04a_en", "retrieved_id": "d741fd7731d3f65bf4fc_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE EXTRACT(MONTH FROM created_at) = 11;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7a33de7fd7159808b04a_noisy_en", "retrieved_id": "4b0f48aca0308fb03bdd_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE EXTRACT(MONTH FROM created_at) = 11;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7a33de7fd7159808b04a_hi", "retrieved_id": "9c094c71f989e977927a_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE EXTRACT(MONTH FROM CAST(created_at AS DATE)) = 11;\",\"project_id\":\"demo_inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "7a33de7fd7159808b04a_hinglish", "retrieved_id": "dd489d306e08ce7cb500_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE EXTRACT(MONTH FROM CAST(created_at AS DATE)) = 11;\",\"project_id\":\"demo_inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "175564da35a6c139e5d8_en", "retrieved_id": "61b32ae34c4100c8fbe8_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE LOWER(display_name) LIKE LOWER('%North%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "175564da35a6c139e5d8_noisy_en", "retrieved_id": "279c576638ef45226927_concrete_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE LOWER(display_name) LIKE LOWER('%Books%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "175564da35a6c139e5d8_hi", "retrieved_id": "6cfe9815e1862d086657_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE LOWER(display_name) LIKE LOWER('%North%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "175564da35a6c139e5d8_hinglish", "retrieved_id": "09ff5f23be86601d0812_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE LOWER(display_name) LIKE LOWER('%North%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "844b55044ac74adf4493_en", "retrieved_id": "4350cfb9872b973f6709_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "844b55044ac74adf4493_noisy_en", "retrieved_id": "4350cfb9872b973f6709_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "844b55044ac74adf4493_hi", "retrieved_id": "80487a866bf00246a506_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "844b55044ac74adf4493_hinglish", "retrieved_id": "97bd4d1472a75cd5d314_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "63fa94106f0cf26f7da2_en", "retrieved_id": "f469bec9595cd61b59b9_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT specimens.title FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id WHERE specimens_groups.label = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "63fa94106f0cf26f7da2_noisy_en", "retrieved_id": "704876acadbeff7dcafb_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT specimens.title FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id WHERE specimens_groups.label = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "63fa94106f0cf26f7da2_hi", "retrieved_id": "1f64dd4a8cd091acb5ed_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT specimens.title FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id WHERE specimens_groups.label = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "63fa94106f0cf26f7da2_hinglish", "retrieved_id": "f469bec9595cd61b59b9_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT specimens.title FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id WHERE specimens_groups.label = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6c37ae504c7cb31a6ef3_en", "retrieved_id": "bd6b78905a3ad03e04b6_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "6c37ae504c7cb31a6ef3_noisy_en", "retrieved_id": "bd6b78905a3ad03e04b6_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "6c37ae504c7cb31a6ef3_hi", "retrieved_id": "944772e48c9b5194fe1e_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "6c37ae504c7cb31a6ef3_hinglish", "retrieved_id": "5165d04cc5f4626621a5_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "a76838f987782ef2d2ed_en", "retrieved_id": "863d3ef71bbbeba4919e_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "a76838f987782ef2d2ed_noisy_en", "retrieved_id": "632bfa2e24875823150a_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "a76838f987782ef2d2ed_hi", "retrieved_id": "286869979f46e9bbc61c_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "a76838f987782ef2d2ed_hinglish", "retrieved_id": "4253da29b10c8d2d921b_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"books\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "44135783de9bd091bfe5_en", "retrieved_id": "fb890425dce9de189858_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_inspections\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "44135783de9bd091bfe5_noisy_en", "retrieved_id": "25ac9041dfe1446dd870_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"],\"project_id\":\"demo_inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "44135783de9bd091bfe5_hi", "retrieved_id": "427fd58d6660abf6f38a_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"],\"project_id\":\"demo_inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "44135783de9bd091bfe5_hinglish", "retrieved_id": "25ac9041dfe1446dd870_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"],\"project_id\":\"demo_inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "4b73bea165fb5f9555f3_en", "retrieved_id": "ff73314a0982d6281641_noisy_en", "output": "{\"action\":\"call\",\"name\":\"database.describe_table\",\"arguments\":{\"table\":\"books\"}}", "metrics": {"json_valid": true, "schema_valid": false, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false, "error": "Unknown tool"}} +{"id": "4b73bea165fb5f9555f3_noisy_en", "retrieved_id": "03727992a863b145c565_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "4b73bea165fb5f9555f3_hi", "retrieved_id": "26ee7e71ccaee33a6162_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "4b73bea165fb5f9555f3_hinglish", "retrieved_id": "0af23c07619fdf39d636_hinglish", "output": "{\"action\":\"call\",\"name\":\"database.describe_table\",\"arguments\":{\"table\":\"books\"}}", "metrics": {"json_valid": true, "schema_valid": false, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false, "error": "Unknown tool"}} +{"id": "d09477cb849c0af3827d_en", "retrieved_id": "a5928e9d2a9de389c96e_noisy_en", "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "d09477cb849c0af3827d_noisy_en", "retrieved_id": "7b86419573eea700d641_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": false, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false}} +{"id": "d09477cb849c0af3827d_hi", "retrieved_id": "d8fbf98bcb77ca92a390_hi", "output": "{\"action\":\"clarify\",\"question\":\"सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "d09477cb849c0af3827d_hinglish", "retrieved_id": "981c50947db171d5fec5_hinglish", "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "5f07a4c7c387316664f1_en", "retrieved_id": "22a9c838578d2feeaf02_noisy_en", "output": "{\"action\":\"clarify\",\"question\":\"Which value should department equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "5f07a4c7c387316664f1_noisy_en", "retrieved_id": "22a9c838578d2feeaf02_noisy_en", "output": "{\"action\":\"clarify\",\"question\":\"Which value should department equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "5f07a4c7c387316664f1_hi", "retrieved_id": "677a222f59b0e9310e4e_hi", "output": "{\"action\":\"clarify\",\"question\":\"department का कौन सा मान चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "5f07a4c7c387316664f1_hinglish", "retrieved_id": "bbdeaa042d51e3f80dcc_hinglish", "output": "{\"action\":\"clarify\",\"question\":\"department ki kaunsi value chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "ac83eae59947f137269f_en", "retrieved_id": "8fde1669f8ae856382b5_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": false, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false}} +{"id": "ac83eae59947f137269f_noisy_en", "retrieved_id": "f6d1979b8e8273fb994b_en", "output": "{\"action\":\"clarify\",\"question\":\"This connection is read-only. Would you like a SELECT query instead?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "ac83eae59947f137269f_hi", "retrieved_id": "a4f18f2163d8129dfc3c_hi", "output": "{\"action\":\"clarify\",\"question\":\"यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "ac83eae59947f137269f_hinglish", "retrieved_id": "6cad228416bb08b460f6_hinglish", "output": "{\"action\":\"clarify\",\"question\":\"Yeh connection read-only hai. Kya SELECT query chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "635a24ec4bc42b67226a_en", "retrieved_id": "03a2503aa4caeb5b0396_en", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "635a24ec4bc42b67226a_noisy_en", "retrieved_id": "fd5016d36c195ab50c95_noisy_en", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "635a24ec4bc42b67226a_hi", "retrieved_id": "fcc2c16620594edcdeae_hi", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "635a24ec4bc42b67226a_hinglish", "retrieved_id": "c6e7cd6d43a8ae36b611_hinglish", "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "8c8ba16d1f868e7d81a9_en", "retrieved_id": "a927da4da3fb93076b11_en", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"indexes\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "8c8ba16d1f868e7d81a9_noisy_en", "retrieved_id": "b70d3ac89725558f5112_en", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"indexes\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "8c8ba16d1f868e7d81a9_hi", "retrieved_id": "b9a03d467eb83da89c0a_hi", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"indexes\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "8c8ba16d1f868e7d81a9_hinglish", "retrieved_id": "9a27f495c05e46011062_hinglish", "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"indexes\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "0161f123d136665478c9_en", "retrieved_id": "9c9565eadc46c39ab34c_en", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "0161f123d136665478c9_noisy_en", "retrieved_id": "9c9565eadc46c39ab34c_en", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "0161f123d136665478c9_hi", "retrieved_id": "00319aa9372ffea135d7_hi", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "0161f123d136665478c9_hinglish", "retrieved_id": "d53f5c6036055adc1894_hinglish", "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "89c9ecdcb7242ef71d8b_en", "retrieved_id": "7433354d41a6c516f4b6_en", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-8757\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "89c9ecdcb7242ef71d8b_noisy_en", "retrieved_id": "4c422d229cf597a3acdd_noisy_en_curriculum_99606", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-117\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "89c9ecdcb7242ef71d8b_hi", "retrieved_id": "b0c31fd3cb1afd956208_hi", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-9402\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "89c9ecdcb7242ef71d8b_hinglish", "retrieved_id": "6ea26747bfbf3b1aa5e0_hinglish", "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-8757\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "a59c6e6bcec70a1667e5_en", "retrieved_id": "3ea115632ccd82802927_noisy_en", "output": "{\"action\":\"answer\",\"text\":\"The tool returned 35 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "a59c6e6bcec70a1667e5_noisy_en", "retrieved_id": "343a07a516f2856cd395_noisy_en", "output": "{\"action\":\"answer\",\"text\":\"The tool returned 35 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "a59c6e6bcec70a1667e5_hi", "retrieved_id": "3ea115632ccd82802927_hi", "output": "{\"action\":\"answer\",\"text\":\"टूल ने 35 पंक्तियाँ लौटाईं।\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "a59c6e6bcec70a1667e5_hinglish", "retrieved_id": "eba566a1ce2244d2976f_hinglish", "output": "{\"action\":\"answer\",\"text\":\"Tool ne 35 rows return ki.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "5689f342b9bf3175e2fe_en", "retrieved_id": "130f7bdeb1d62731f52f_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5689f342b9bf3175e2fe_noisy_en", "retrieved_id": "88a8a9447d5602b34319_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5689f342b9bf3175e2fe_hi", "retrieved_id": "02501bf0d6866d9d8bf4_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5689f342b9bf3175e2fe_hinglish", "retrieved_id": "484e67e978b01025c01e_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "860c064ccde26d474c46_en", "retrieved_id": "427c52f3cda588a8dfa5_concrete_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT DISTINCT kind FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "860c064ccde26d474c46_noisy_en", "retrieved_id": "b2757aefcc8bc0a7549e_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "860c064ccde26d474c46_hi", "retrieved_id": "3aac2f1448f136869091_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "860c064ccde26d474c46_hinglish", "retrieved_id": "6eff07d389c9dffd14ce_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9eee465dd6b4c3a090c6_en", "retrieved_id": "409932933bfc69fcd733_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE region = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9eee465dd6b4c3a090c6_noisy_en", "retrieved_id": "64a84825485089e8ab92_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE region = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9eee465dd6b4c3a090c6_hi", "retrieved_id": "8d0307bba5b528e9d66d_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE region IN ('Pune', 'Pune');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "9eee465dd6b4c3a090c6_hinglish", "retrieved_id": "64a84825485089e8ab92_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE region = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "dd1037fa65f5f5a2a06f_en", "retrieved_id": "e7a8721fe66276736a60_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT customer_name FROM reservations WHERE department = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "dd1037fa65f5f5a2a06f_noisy_en", "retrieved_id": "e7a8721fe66276736a60_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT customer_name FROM reservations WHERE department = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "dd1037fa65f5f5a2a06f_hi", "retrieved_id": "42ebadd189f34f44115e_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT customer_name FROM reservations WHERE department = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "dd1037fa65f5f5a2a06f_hinglish", "retrieved_id": "26864a1bafb0472f505d_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT customer_name FROM reservations WHERE department = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "dddd09f34c0e849282a8_en", "retrieved_id": "4e2d141c0e620910423a_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE salary <= 20;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "dddd09f34c0e849282a8_noisy_en", "retrieved_id": "f27668acc136794b63af_concrete_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE salary > 10000;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "dddd09f34c0e849282a8_hi", "retrieved_id": "93630964314d2a1070ff_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE item_name IS NULL;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "dddd09f34c0e849282a8_hinglish", "retrieved_id": "b900a96f38adb3290f5c_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE salary < 20;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "0ce22f4c8bc7554d586d_en", "retrieved_id": "b2684d967d976714b34c_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE score < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0ce22f4c8bc7554d586d_noisy_en", "retrieved_id": "bb74a6cb1307da5e5652_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE score < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0ce22f4c8bc7554d586d_hi", "retrieved_id": "6c50e81ff0658a9ae684_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE score < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0ce22f4c8bc7554d586d_hinglish", "retrieved_id": "b2684d967d976714b34c_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE score < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e02d95c6aba4ed336a6a_en", "retrieved_id": "cfd917d72e37aed08030_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE cost > 50;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "e02d95c6aba4ed336a6a_noisy_en", "retrieved_id": "3dab7c0c56fc690e80c6_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE cost < 50;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "e02d95c6aba4ed336a6a_hi", "retrieved_id": "3dab7c0c56fc690e80c6_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE cost < 50;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "e02d95c6aba4ed336a6a_hinglish", "retrieved_id": "3285d0ab97b5b4b5a64a_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE cost >= 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7e281ac4ee7d2d835606_en", "retrieved_id": "5264a22783f37a6feb83_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE rating >= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "7e281ac4ee7d2d835606_noisy_en", "retrieved_id": "1d162ce454d0bc0f4904_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE rating <= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7e281ac4ee7d2d835606_hi", "retrieved_id": "55443a8bb63ef75dfd2c_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE rating >= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "7e281ac4ee7d2d835606_hinglish", "retrieved_id": "2c35c7809e5b03ac98ce_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE rating <= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "dc59f00f50b6c8f7bc16_en", "retrieved_id": "3cdcd111a3c76ece217a_concrete_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE amount BETWEEN 20 AND 520;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "dc59f00f50b6c8f7bc16_noisy_en", "retrieved_id": "3cdcd111a3c76ece217a_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE amount BETWEEN 20 AND 520;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "dc59f00f50b6c8f7bc16_hi", "retrieved_id": "3cdcd111a3c76ece217a_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE amount BETWEEN 20 AND 520;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "dc59f00f50b6c8f7bc16_hinglish", "retrieved_id": "fdf0110a5d2ced536788_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE amount BETWEEN 20 AND 520;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b2725cc841b5aaa2fba6_en", "retrieved_id": "a026b93f01efebee9682_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE kind = 'Electronics' AND price > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b2725cc841b5aaa2fba6_noisy_en", "retrieved_id": "af2caf996b68c0c6c8db_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE kind = 'O''Reilly' AND price > 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "b2725cc841b5aaa2fba6_hi", "retrieved_id": "825409f04505f8193411_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE kind = 'Pune' AND price > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "b2725cc841b5aaa2fba6_hinglish", "retrieved_id": "825409f04505f8193411_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE kind = 'Pune' AND price > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "4f5c410b4b3379c58420_en", "retrieved_id": "26976bb271af313d9a97_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE category IN ('pending', 'North');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4f5c410b4b3379c58420_noisy_en", "retrieved_id": "78ec50c1f7496cff5848_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE category IN ('North', 'pending');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "4f5c410b4b3379c58420_hi", "retrieved_id": "78ec50c1f7496cff5848_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE category IN ('North', 'pending');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "4f5c410b4b3379c58420_hinglish", "retrieved_id": "78ec50c1f7496cff5848_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE category IN ('North', 'pending');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "a6079b58d4365a8f70c8_en", "retrieved_id": "bda2d640bb8440ba316d_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a6079b58d4365a8f70c8_noisy_en", "retrieved_id": "2979500f0ecfcf0ceeaf_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a6079b58d4365a8f70c8_hi", "retrieved_id": "6f58fb9ba5d390237de3_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE NOT title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "a6079b58d4365a8f70c8_hinglish", "retrieved_id": "5bb801582af241e0e0d2_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE NOT title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "e71814820a36875f0ee5_en", "retrieved_id": "993d6dbb886fdcff8483_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE NOT description IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "e71814820a36875f0ee5_noisy_en", "retrieved_id": "52c6084dc05c08ecedad_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE description IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "e71814820a36875f0ee5_hi", "retrieved_id": "36846cba4b79e7ab511e_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE description IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e71814820a36875f0ee5_hinglish", "retrieved_id": "006e0510328e4f2c7c4a_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE description IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "78fd6e66c9d726084543_en", "retrieved_id": "c40602913c502e7ccda9_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM exhibits WHERE kind = 'Mumbai';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "78fd6e66c9d726084543_noisy_en", "retrieved_id": "6040c6883947ec5b3821_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "78fd6e66c9d726084543_hi", "retrieved_id": "2d416c705002edc7a1af_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "78fd6e66c9d726084543_hinglish", "retrieved_id": "33e0077bee491c810f9d_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7e1cd79fe94cad9c7a9e_en", "retrieved_id": "cbc3795c0ca21a9b8c5a_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "7e1cd79fe94cad9c7a9e_noisy_en", "retrieved_id": "f237dec70fbabb886e7f_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "7e1cd79fe94cad9c7a9e_hi", "retrieved_id": "c2de4d9083ab590afc32_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE region IN ('active', 'Books');\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "7e1cd79fe94cad9c7a9e_hinglish", "retrieved_id": "c2f4b5de63f76a49c82d_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT region, COUNT(*) FROM specimens GROUP BY region;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "5ce4cc7ea9d93a0aa4c5_en", "retrieved_id": "34916d4afc7c006b63d7_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT department, SUM(quantity) FROM reservations GROUP BY department;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "5ce4cc7ea9d93a0aa4c5_noisy_en", "retrieved_id": "be20ec0376eee65aa2ce_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(quantity) FROM reservations;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5ce4cc7ea9d93a0aa4c5_hi", "retrieved_id": "3393ec0013dc7be8e9e1_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT SUM(quantity) FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5ce4cc7ea9d93a0aa4c5_hinglish", "retrieved_id": "c530368611b81eb63763_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(quantity) FROM reservations;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a4cdc86496ddd0c1cc90_en", "retrieved_id": "4dd068aa920034a474f8_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT AVG(cost) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a4cdc86496ddd0c1cc90_noisy_en", "retrieved_id": "b2121f866ae29081d2a0_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT AVG(cost) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a4cdc86496ddd0c1cc90_hi", "retrieved_id": "697d95a95d9ba4d44f0e_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT AVG(cost) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a4cdc86496ddd0c1cc90_hinglish", "retrieved_id": "18cdefae957dc6d12228_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT AVG(cost) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ac6cc6f05e6d013971e5_en", "retrieved_id": "747ca8e86fd928154f9c_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT MAX(quantity) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ac6cc6f05e6d013971e5_noisy_en", "retrieved_id": "e58b057a43bf930f66c1_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(quantity) FROM specimens;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ac6cc6f05e6d013971e5_hi", "retrieved_id": "b6185f615e05ea951f2f_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(quantity) FROM specimens;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "ac6cc6f05e6d013971e5_hinglish", "retrieved_id": "8791137d1c7a2875f0db_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(quantity) FROM specimens;\",\"project_id\":\"demo_specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "aeeb76897a32b9fd2df4_en", "retrieved_id": "4d035a536476fdd62299_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT MIN(amount) FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "aeeb76897a32b9fd2df4_noisy_en", "retrieved_id": "2191ec0e7a2e2e4e18b2_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(amount) FROM reservations;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "aeeb76897a32b9fd2df4_hi", "retrieved_id": "60337122c76970ce6f93_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(amount) FROM reservations;\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "aeeb76897a32b9fd2df4_hinglish", "retrieved_id": "9a6db088671e3d6461b8_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT MIN(amount) FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "508d2834a4743665db1f_en", "retrieved_id": "8324f0ab6af132deba22_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT DISTINCT kind FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "508d2834a4743665db1f_noisy_en", "retrieved_id": "8324f0ab6af132deba22_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT DISTINCT kind FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "508d2834a4743665db1f_hi", "retrieved_id": "70873838e6be879c89a1_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT DISTINCT kind FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "508d2834a4743665db1f_hinglish", "retrieved_id": "f0e8a1aa17ebe2943f9a_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE display_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "7fb4f6e9f1b68e05cfd9_en", "retrieved_id": "de723244833eb9acd9b0_concrete_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY CASE WHEN salary IS NULL THEN 1 ELSE 0 END DESC, salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "7fb4f6e9f1b68e05cfd9_noisy_en", "retrieved_id": "dbbc87df17789dcf16c7_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7fb4f6e9f1b68e05cfd9_hi", "retrieved_id": "5f714fc20f63b02a2389_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7fb4f6e9f1b68e05cfd9_hinglish", "retrieved_id": "9360abd24d181d80b919_concrete_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "02981b7ccddb9b467260_en", "retrieved_id": "9893daa34daf0837aacd_concrete_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits ORDER BY CASE WHEN quantity IS NULL THEN 1 ELSE 0 END DESC, quantity DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "02981b7ccddb9b467260_noisy_en", "retrieved_id": "08cdefb37331da7aacab_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits ORDER BY quantity DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "02981b7ccddb9b467260_hi", "retrieved_id": "e31c637236043ab2d50e_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits ORDER BY quantity ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "02981b7ccddb9b467260_hinglish", "retrieved_id": "bd6dcbc8d4b468bfdc18_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits ORDER BY quantity ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9956c110975a38c3d293_en", "retrieved_id": "a62d054fdafb7835efc0_concrete_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY CASE WHEN cost IS NULL THEN 1 ELSE 0 END, cost ASC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "9956c110975a38c3d293_noisy_en", "retrieved_id": "34db1cbffb2c6826f7d1_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY CASE WHEN cost IS NULL THEN 1 ELSE 0 END DESC, cost DESC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "9956c110975a38c3d293_hi", "retrieved_id": "36a2047a9b8d5e928dc2_concrete_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY cost DESC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9956c110975a38c3d293_hinglish", "retrieved_id": "912be65554b401892c89_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY CASE WHEN cost IS NULL THEN 1 ELSE 0 END DESC, cost DESC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "785d62b3d5e4932f5c33_en", "retrieved_id": "797a7cee699e825b8d9d_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY rating ASC LIMIT 20;\",\"project_id\":\"demo_inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "785d62b3d5e4932f5c33_noisy_en", "retrieved_id": "797a7cee699e825b8d9d_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY rating ASC LIMIT 20;\",\"project_id\":\"demo_inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "785d62b3d5e4932f5c33_hi", "retrieved_id": "3a5ecd2a0421c0799bf9_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections ORDER BY rating ASC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "785d62b3d5e4932f5c33_hinglish", "retrieved_id": "797a7cee699e825b8d9d_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY rating ASC LIMIT 20;\",\"project_id\":\"demo_inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "253e89e4cdbbd5c2d0f9_en", "retrieved_id": "a103c6d9dacd84da241e_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT status, COUNT(*) FROM specimens GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "253e89e4cdbbd5c2d0f9_noisy_en", "retrieved_id": "51e52f3bb43aa0c6175a_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT status, COUNT(*) FROM specimens GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "253e89e4cdbbd5c2d0f9_hi", "retrieved_id": "cdec67016dd24ea79601_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT status, COUNT(*) FROM specimens GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "253e89e4cdbbd5c2d0f9_hinglish", "retrieved_id": "8dd5eeb5da69f3f2663f_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT status, COUNT(*) FROM specimens GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ec5a4c33f3a4c1fdfef3_en", "retrieved_id": "665c5ccec5c32f5bd132_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT city, SUM(rating) FROM reservations GROUP BY city;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ec5a4c33f3a4c1fdfef3_noisy_en", "retrieved_id": "8db52bac8a3a32a185cc_concrete_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT city, SUM(rating) FROM reservations GROUP BY city;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ec5a4c33f3a4c1fdfef3_hi", "retrieved_id": "665c5ccec5c32f5bd132_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT city, SUM(rating) FROM reservations GROUP BY city;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ec5a4c33f3a4c1fdfef3_hinglish", "retrieved_id": "5e531feca720b48f642d_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT city, SUM(rating) FROM reservations GROUP BY city;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0abe13c841fd7c3ac885_en", "retrieved_id": "d157bf54f9477adfe748_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT region, COUNT(*) FROM inspections GROUP BY region HAVING COUNT(*) > 5000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0abe13c841fd7c3ac885_noisy_en", "retrieved_id": "d157bf54f9477adfe748_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT region, COUNT(*) FROM inspections GROUP BY region HAVING COUNT(*) > 5000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0abe13c841fd7c3ac885_hi", "retrieved_id": "d8178d83cd37b65bada0_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT region, COUNT(*) FROM inspections GROUP BY region HAVING COUNT(*) > 5000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0abe13c841fd7c3ac885_hinglish", "retrieved_id": "a790a59140eddc41a1f0_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT region, COUNT(*) FROM inspections GROUP BY region HAVING COUNT(*) > 5000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "41ea5651dda516a4ac30_en", "retrieved_id": "8f9ade63176f38ee0661_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE order_date >= '2023-12-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "41ea5651dda516a4ac30_noisy_en", "retrieved_id": "8f9ade63176f38ee0661_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE order_date >= '2023-12-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "41ea5651dda516a4ac30_hi", "retrieved_id": "a3f4045fd93493edf1c9_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE order_date >= '2023-12-15';\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "41ea5651dda516a4ac30_hinglish", "retrieved_id": "fbe19ee8a1e10f18cd09_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE order_date >= '2023-02-15';\",\"project_id\":\"demo_reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "1d7ab92421915fd47ebe_en", "retrieved_id": "fa668f5f74b7998e4689_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE YEAR(created_at) = 2024;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1d7ab92421915fd47ebe_noisy_en", "retrieved_id": "774eabfe1a00326f842e_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE YEAR(created_at) = 2024;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1d7ab92421915fd47ebe_hi", "retrieved_id": "52c736038a7903fa3560_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE YEAR(created_at) = 2024;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1d7ab92421915fd47ebe_hinglish", "retrieved_id": "4a29dc069452fce48052_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE EXTRACT(YEAR FROM created_at) = 2024;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "9f8fa242b083f38bc5f8_en", "retrieved_id": "7a828c960cef15c5203d_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE EXTRACT(MONTH FROM order_date) = 4;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": true}} +{"id": "9f8fa242b083f38bc5f8_noisy_en", "retrieved_id": "cd48e2c7bcbdd06bd806_noisy_en", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE MONTH(order_date) = 4;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9f8fa242b083f38bc5f8_hi", "retrieved_id": "438b46c0411da02e2fd5_hi", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE MONTH(order_date) = 4;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9f8fa242b083f38bc5f8_hinglish", "retrieved_id": "c3a6d1d66a95fc81bdb3_hinglish", "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE MONTH(order_date) = 4;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} diff --git a/evaluation/baseline-test.summary.json b/evaluation/baseline-test.summary.json new file mode 100644 index 0000000000000000000000000000000000000000..8a3bf63cd94168b92d6183673d05013ea3727849 --- /dev/null +++ b/evaluation/baseline-test.summary.json @@ -0,0 +1,1932 @@ +{ + "baseline": "TF-IDF nearest question plus tool-description and DDL-position binding; no expected answer or gold slots used", + "training_questions": 212073, + "examples": 1196, + "seconds": 9.818428039550781, + "metrics": { + "json_valid": { + "correct": 1196, + "total": 1196, + "rate": 1.0 + }, + "schema_valid": { + "correct": 1187, + "total": 1196, + "rate": 0.9924749163879598 + }, + "action_correct": { + "correct": 1184, + "total": 1196, + "rate": 0.9899665551839465 + }, + "tool_correct": { + "correct": 1077, + "total": 1100, + "rate": 0.9790909090909091 + }, + "arguments_exact": { + "correct": 753, + "total": 1100, + "rate": 0.6845454545454546 + }, + "sql_equivalent": { + "correct": 664, + "total": 912, + "rate": 0.7280701754385965 + }, + "success": { + "correct": 881, + "total": 1196, + "rate": 0.7366220735785953 + } + }, + "groups": { + "language:en": { + "json_valid": { + "correct": 299, + "total": 299, + "rate": 1.0 + }, + "schema_valid": { + "correct": 296, + "total": 299, + "rate": 0.9899665551839465 + }, + "action_correct": { + "correct": 293, + "total": 299, + "rate": 0.979933110367893 + }, + "tool_correct": { + "correct": 268, + "total": 275, + "rate": 0.9745454545454545 + }, + "arguments_exact": { + "correct": 169, + "total": 275, + "rate": 0.6145454545454545 + }, + "sql_equivalent": { + "correct": 146, + "total": 228, + "rate": 0.6403508771929824 + }, + "success": { + "correct": 197, + "total": 299, + "rate": 0.6588628762541806 + } + }, + "backend:mysql": { + "json_valid": { + "correct": 608, + "total": 608, + "rate": 1.0 + }, + "schema_valid": { + "correct": 608, + "total": 608, + "rate": 1.0 + }, + "action_correct": { + "correct": 601, + "total": 608, + "rate": 0.9884868421052632 + }, + "tool_correct": { + "correct": 543, + "total": 556, + "rate": 0.9766187050359713 + }, + "arguments_exact": { + "correct": 391, + "total": 556, + "rate": 0.7032374100719424 + }, + "sql_equivalent": { + "correct": 351, + "total": 476, + "rate": 0.7373949579831933 + }, + "success": { + "correct": 452, + "total": 608, + "rate": 0.743421052631579 + } + }, + "operation:all": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "language:noisy_en": { + "json_valid": { + "correct": 299, + "total": 299, + "rate": 1.0 + }, + "schema_valid": { + "correct": 299, + "total": 299, + "rate": 1.0 + }, + "action_correct": { + "correct": 293, + "total": 299, + "rate": 0.979933110367893 + }, + "tool_correct": { + "correct": 270, + "total": 275, + "rate": 0.9818181818181818 + }, + "arguments_exact": { + "correct": 195, + "total": 275, + "rate": 0.7090909090909091 + }, + "sql_equivalent": { + "correct": 166, + "total": 228, + "rate": 0.7280701754385965 + }, + "success": { + "correct": 221, + "total": 299, + "rate": 0.7391304347826086 + } + }, + "language:hi": { + "json_valid": { + "correct": 299, + "total": 299, + "rate": 1.0 + }, + "schema_valid": { + "correct": 299, + "total": 299, + "rate": 1.0 + }, + "action_correct": { + "correct": 299, + "total": 299, + "rate": 1.0 + }, + "tool_correct": { + "correct": 270, + "total": 275, + "rate": 0.9818181818181818 + }, + "arguments_exact": { + "correct": 183, + "total": 275, + "rate": 0.6654545454545454 + }, + "sql_equivalent": { + "correct": 162, + "total": 228, + "rate": 0.7105263157894737 + }, + "success": { + "correct": 222, + "total": 299, + "rate": 0.7424749163879598 + } + }, + "language:hinglish": { + "json_valid": { + "correct": 299, + "total": 299, + "rate": 1.0 + }, + "schema_valid": { + "correct": 293, + "total": 299, + "rate": 0.979933110367893 + }, + "action_correct": { + "correct": 299, + "total": 299, + "rate": 1.0 + }, + "tool_correct": { + "correct": 269, + "total": 275, + "rate": 0.9781818181818182 + }, + "arguments_exact": { + "correct": 206, + "total": 275, + "rate": 0.7490909090909091 + }, + "sql_equivalent": { + "correct": 190, + "total": 228, + "rate": 0.8333333333333334 + }, + "success": { + "correct": 241, + "total": 299, + "rate": 0.8060200668896321 + } + }, + "backend:supabase": { + "json_valid": { + "correct": 588, + "total": 588, + "rate": 1.0 + }, + "schema_valid": { + "correct": 579, + "total": 588, + "rate": 0.9846938775510204 + }, + "action_correct": { + "correct": 583, + "total": 588, + "rate": 0.9914965986394558 + }, + "tool_correct": { + "correct": 534, + "total": 544, + "rate": 0.9816176470588235 + }, + "arguments_exact": { + "correct": 362, + "total": 544, + "rate": 0.6654411764705882 + }, + "sql_equivalent": { + "correct": 313, + "total": 436, + "rate": 0.7178899082568807 + }, + "success": { + "correct": 429, + "total": 588, + "rate": 0.7295918367346939 + } + }, + "operation:project": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 21, + "total": 28, + "rate": 0.75 + }, + "sql_equivalent": { + "correct": 21, + "total": 28, + "rate": 0.75 + }, + "success": { + "correct": 21, + "total": 28, + "rate": 0.75 + } + }, + "operation:eq": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 13, + "total": 28, + "rate": 0.4642857142857143 + }, + "sql_equivalent": { + "correct": 14, + "total": 28, + "rate": 0.5 + }, + "success": { + "correct": 14, + "total": 28, + "rate": 0.5 + } + }, + "operation:project_eq": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 26, + "total": 28, + "rate": 0.9285714285714286 + }, + "sql_equivalent": { + "correct": 26, + "total": 28, + "rate": 0.9285714285714286 + }, + "success": { + "correct": 26, + "total": 28, + "rate": 0.9285714285714286 + } + }, + "operation:gt": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 3, + "total": 28, + "rate": 0.10714285714285714 + }, + "sql_equivalent": { + "correct": 3, + "total": 28, + "rate": 0.10714285714285714 + }, + "success": { + "correct": 3, + "total": 28, + "rate": 0.10714285714285714 + } + }, + "operation:lt": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 27, + "total": 28, + "rate": 0.9642857142857143 + }, + "sql_equivalent": { + "correct": 27, + "total": 28, + "rate": 0.9642857142857143 + }, + "success": { + "correct": 27, + "total": 28, + "rate": 0.9642857142857143 + } + }, + "operation:gte": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 11, + "total": 28, + "rate": 0.39285714285714285 + }, + "sql_equivalent": { + "correct": 11, + "total": 28, + "rate": 0.39285714285714285 + }, + "success": { + "correct": 11, + "total": 28, + "rate": 0.39285714285714285 + } + }, + "operation:lte": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 13, + "total": 28, + "rate": 0.4642857142857143 + }, + "sql_equivalent": { + "correct": 13, + "total": 28, + "rate": 0.4642857142857143 + }, + "success": { + "correct": 13, + "total": 28, + "rate": 0.4642857142857143 + } + }, + "operation:between": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 24, + "total": 28, + "rate": 0.8571428571428571 + }, + "sql_equivalent": { + "correct": 24, + "total": 28, + "rate": 0.8571428571428571 + }, + "success": { + "correct": 24, + "total": 28, + "rate": 0.8571428571428571 + } + }, + "operation:and": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 16, + "total": 28, + "rate": 0.5714285714285714 + }, + "sql_equivalent": { + "correct": 16, + "total": 28, + "rate": 0.5714285714285714 + }, + "success": { + "correct": 16, + "total": 28, + "rate": 0.5714285714285714 + } + }, + "operation:or": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 13, + "total": 28, + "rate": 0.4642857142857143 + }, + "sql_equivalent": { + "correct": 23, + "total": 28, + "rate": 0.8214285714285714 + }, + "success": { + "correct": 23, + "total": 28, + "rate": 0.8214285714285714 + } + }, + "operation:null": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 18, + "total": 28, + "rate": 0.6428571428571429 + }, + "sql_equivalent": { + "correct": 18, + "total": 28, + "rate": 0.6428571428571429 + }, + "success": { + "correct": 18, + "total": 28, + "rate": 0.6428571428571429 + } + }, + "operation:not_null": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 14, + "total": 28, + "rate": 0.5 + }, + "sql_equivalent": { + "correct": 21, + "total": 28, + "rate": 0.75 + }, + "success": { + "correct": 21, + "total": 28, + "rate": 0.75 + } + }, + "operation:count": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 21, + "total": 28, + "rate": 0.75 + }, + "sql_equivalent": { + "correct": 21, + "total": 28, + "rate": 0.75 + }, + "success": { + "correct": 21, + "total": 28, + "rate": 0.75 + } + }, + "operation:count_eq": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 13, + "total": 28, + "rate": 0.4642857142857143 + }, + "sql_equivalent": { + "correct": 13, + "total": 28, + "rate": 0.4642857142857143 + }, + "success": { + "correct": 13, + "total": 28, + "rate": 0.4642857142857143 + } + }, + "operation:sum": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 27, + "total": 28, + "rate": 0.9642857142857143 + }, + "arguments_exact": { + "correct": 21, + "total": 28, + "rate": 0.75 + }, + "sql_equivalent": { + "correct": 21, + "total": 28, + "rate": 0.75 + }, + "success": { + "correct": 21, + "total": 28, + "rate": 0.75 + } + }, + "operation:avg": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:max": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 16, + "total": 28, + "rate": 0.5714285714285714 + }, + "sql_equivalent": { + "correct": 16, + "total": 28, + "rate": 0.5714285714285714 + }, + "success": { + "correct": 16, + "total": 28, + "rate": 0.5714285714285714 + } + }, + "operation:min": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:distinct": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 21, + "total": 28, + "rate": 0.75 + }, + "sql_equivalent": { + "correct": 21, + "total": 28, + "rate": 0.75 + }, + "success": { + "correct": 21, + "total": 28, + "rate": 0.75 + } + }, + "operation:sort_desc": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 12, + "total": 28, + "rate": 0.42857142857142855 + }, + "sql_equivalent": { + "correct": 12, + "total": 28, + "rate": 0.42857142857142855 + }, + "success": { + "correct": 12, + "total": 28, + "rate": 0.42857142857142855 + } + }, + "operation:sort_asc": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 12, + "total": 28, + "rate": 0.42857142857142855 + }, + "sql_equivalent": { + "correct": 12, + "total": 28, + "rate": 0.42857142857142855 + }, + "success": { + "correct": 12, + "total": 28, + "rate": 0.42857142857142855 + } + }, + "operation:top": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 8, + "total": 28, + "rate": 0.2857142857142857 + }, + "sql_equivalent": { + "correct": 8, + "total": 28, + "rate": 0.2857142857142857 + }, + "success": { + "correct": 8, + "total": 28, + "rate": 0.2857142857142857 + } + }, + "operation:bottom": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 17, + "total": 28, + "rate": 0.6071428571428571 + }, + "sql_equivalent": { + "correct": 17, + "total": 28, + "rate": 0.6071428571428571 + }, + "success": { + "correct": 17, + "total": 28, + "rate": 0.6071428571428571 + } + }, + "operation:group_count": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:group_sum": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:having": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 25, + "total": 28, + "rate": 0.8928571428571429 + }, + "sql_equivalent": { + "correct": 25, + "total": 28, + "rate": 0.8928571428571429 + }, + "success": { + "correct": 25, + "total": 28, + "rate": 0.8928571428571429 + } + }, + "operation:date_after": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 19, + "total": 28, + "rate": 0.6785714285714286 + }, + "sql_equivalent": { + "correct": 22, + "total": 28, + "rate": 0.7857142857142857 + }, + "success": { + "correct": 22, + "total": 28, + "rate": 0.7857142857142857 + } + }, + "operation:year": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 13, + "total": 28, + "rate": 0.4642857142857143 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:month": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 12, + "total": 28, + "rate": 0.42857142857142855 + }, + "sql_equivalent": { + "correct": 26, + "total": 28, + "rate": 0.9285714285714286 + }, + "success": { + "correct": 26, + "total": 28, + "rate": 0.9285714285714286 + } + }, + "operation:contains": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 19, + "total": 24, + "rate": 0.7916666666666666 + }, + "sql_equivalent": { + "correct": 19, + "total": 24, + "rate": 0.7916666666666666 + }, + "success": { + "correct": 19, + "total": 24, + "rate": 0.7916666666666666 + } + }, + "operation:join": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "success": { + "correct": 24, + "total": 24, + "rate": 1.0 + } + }, + "operation:join_filter": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 22, + "total": 24, + "rate": 0.9166666666666666 + }, + "sql_equivalent": { + "correct": 22, + "total": 24, + "rate": 0.9166666666666666 + }, + "success": { + "correct": 22, + "total": 24, + "rate": 0.9166666666666666 + } + }, + "operation:list_tables": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 24, + "total": 24, + "rate": 1.0 + } + }, + "operation:describe": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 21, + "total": 24, + "rate": 0.875 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 12, + "total": 24, + "rate": 0.5 + }, + "arguments_exact": { + "correct": 9, + "total": 24, + "rate": 0.375 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 9, + "total": 24, + "rate": 0.375 + } + }, + "operation:missing_schema": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 24, + "total": 24, + "rate": 1.0 + } + }, + "operation:sql_error": { + "json_valid": { + "correct": 20, + "total": 20, + "rate": 1.0 + }, + "schema_valid": { + "correct": 14, + "total": 20, + "rate": 0.7 + }, + "action_correct": { + "correct": 20, + "total": 20, + "rate": 1.0 + }, + "tool_correct": { + "correct": 10, + "total": 20, + "rate": 0.5 + }, + "arguments_exact": { + "correct": 6, + "total": 20, + "rate": 0.3 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 6, + "total": 20, + "rate": 0.3 + } + }, + "operation:ambiguous": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 18, + "total": 24, + "rate": 0.75 + }, + "tool_correct": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "arguments_exact": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 18, + "total": 24, + "rate": 0.75 + } + }, + "operation:missing_value": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "arguments_exact": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 24, + "total": 24, + "rate": 1.0 + } + }, + "operation:write": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 18, + "total": 24, + "rate": 0.75 + }, + "tool_correct": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "arguments_exact": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 18, + "total": 24, + "rate": 0.75 + } + }, + "operation:weather": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 24, + "total": 24, + "rate": 1.0 + } + }, + "operation:search": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 24, + "total": 24, + "rate": 1.0 + } + }, + "operation:read_file": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 24, + "total": 24, + "rate": 1.0 + } + }, + "operation:ticket": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 4, + "total": 24, + "rate": 0.16666666666666666 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 4, + "total": 24, + "rate": 0.16666666666666666 + } + }, + "operation:final": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "arguments_exact": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 18, + "total": 24, + "rate": 0.75 + } + } + } +} \ No newline at end of file diff --git a/evaluation/development.summary.json b/evaluation/development.summary.json new file mode 100644 index 0000000000000000000000000000000000000000..1fe8d62e0f0816fc96f4f79a93943d609ad572bf --- /dev/null +++ b/evaluation/development.summary.json @@ -0,0 +1,1785 @@ +{ + "examples": 192, + "seconds": 6.547292470932007, + "decoding": "raw greedy, no repair, no teacher fallback", + "checkpoint_sha256": "9489b5cde69c11f64b3e7031182e8248aa83790f1ad4aeefbf99a4e5e152e692", + "sql_metric": "Dialect parsing and equivalent results on two generated SQLite fixtures after dialect adaptation; not native MySQL/PostgreSQL execution.", + "metrics": { + "json_valid": { + "correct": 192, + "total": 192, + "rate": 1.0 + }, + "schema_valid": { + "correct": 192, + "total": 192, + "rate": 1.0 + }, + "action_correct": { + "correct": 192, + "total": 192, + "rate": 1.0 + }, + "tool_correct": { + "correct": 192, + "total": 192, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 188, + "total": 192, + "rate": 0.9791666666666666 + }, + "sql_equivalent": { + "correct": 125, + "total": 128, + "rate": 0.9765625 + }, + "success": { + "correct": 188, + "total": 192, + "rate": 0.9791666666666666 + } + }, + "groups": { + "language:noisy_en": { + "json_valid": { + "correct": 48, + "total": 48, + "rate": 1.0 + }, + "schema_valid": { + "correct": 48, + "total": 48, + "rate": 1.0 + }, + "action_correct": { + "correct": 48, + "total": 48, + "rate": 1.0 + }, + "tool_correct": { + "correct": 48, + "total": 48, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 48, + "total": 48, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 31, + "total": 31, + "rate": 1.0 + }, + "success": { + "correct": 48, + "total": 48, + "rate": 1.0 + } + }, + "backend:supabase": { + "json_valid": { + "correct": 92, + "total": 92, + "rate": 1.0 + }, + "schema_valid": { + "correct": 92, + "total": 92, + "rate": 1.0 + }, + "action_correct": { + "correct": 92, + "total": 92, + "rate": 1.0 + }, + "tool_correct": { + "correct": 92, + "total": 92, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 90, + "total": 92, + "rate": 0.9782608695652174 + }, + "sql_equivalent": { + "correct": 64, + "total": 66, + "rate": 0.9696969696969697 + }, + "success": { + "correct": 90, + "total": 92, + "rate": 0.9782608695652174 + } + }, + "operation:missing_schema": { + "json_valid": { + "correct": 7, + "total": 7, + "rate": 1.0 + }, + "schema_valid": { + "correct": 7, + "total": 7, + "rate": 1.0 + }, + "action_correct": { + "correct": 7, + "total": 7, + "rate": 1.0 + }, + "tool_correct": { + "correct": 7, + "total": 7, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 7, + "total": 7, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 7, + "total": 7, + "rate": 1.0 + } + }, + "operation:project": { + "json_valid": { + "correct": 7, + "total": 7, + "rate": 1.0 + }, + "schema_valid": { + "correct": 7, + "total": 7, + "rate": 1.0 + }, + "action_correct": { + "correct": 7, + "total": 7, + "rate": 1.0 + }, + "tool_correct": { + "correct": 7, + "total": 7, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 6, + "total": 7, + "rate": 0.8571428571428571 + }, + "sql_equivalent": { + "correct": 6, + "total": 7, + "rate": 0.8571428571428571 + }, + "success": { + "correct": 6, + "total": 7, + "rate": 0.8571428571428571 + } + }, + "language:hinglish": { + "json_valid": { + "correct": 52, + "total": 52, + "rate": 1.0 + }, + "schema_valid": { + "correct": 52, + "total": 52, + "rate": 1.0 + }, + "action_correct": { + "correct": 52, + "total": 52, + "rate": 1.0 + }, + "tool_correct": { + "correct": 52, + "total": 52, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 51, + "total": 52, + "rate": 0.9807692307692307 + }, + "sql_equivalent": { + "correct": 36, + "total": 37, + "rate": 0.972972972972973 + }, + "success": { + "correct": 51, + "total": 52, + "rate": 0.9807692307692307 + } + }, + "operation:count_eq": { + "json_valid": { + "correct": 5, + "total": 5, + "rate": 1.0 + }, + "schema_valid": { + "correct": 5, + "total": 5, + "rate": 1.0 + }, + "action_correct": { + "correct": 5, + "total": 5, + "rate": 1.0 + }, + "tool_correct": { + "correct": 5, + "total": 5, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 5, + "total": 5, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 5, + "total": 5, + "rate": 1.0 + }, + "success": { + "correct": 5, + "total": 5, + "rate": 1.0 + } + }, + "language:en": { + "json_valid": { + "correct": 44, + "total": 44, + "rate": 1.0 + }, + "schema_valid": { + "correct": 44, + "total": 44, + "rate": 1.0 + }, + "action_correct": { + "correct": 44, + "total": 44, + "rate": 1.0 + }, + "tool_correct": { + "correct": 44, + "total": 44, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 42, + "total": 44, + "rate": 0.9545454545454546 + }, + "sql_equivalent": { + "correct": 30, + "total": 31, + "rate": 0.967741935483871 + }, + "success": { + "correct": 42, + "total": 44, + "rate": 0.9545454545454546 + } + }, + "operation:group_count": { + "json_valid": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "schema_valid": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "action_correct": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "tool_correct": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "success": { + "correct": 3, + "total": 3, + "rate": 1.0 + } + }, + "operation:distinct": { + "json_valid": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "schema_valid": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "action_correct": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "tool_correct": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "success": { + "correct": 4, + "total": 4, + "rate": 1.0 + } + }, + "operation:year": { + "json_valid": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "schema_valid": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "action_correct": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "tool_correct": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "success": { + "correct": 4, + "total": 4, + "rate": 1.0 + } + }, + "operation:max": { + "json_valid": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "schema_valid": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "action_correct": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "tool_correct": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "success": { + "correct": 3, + "total": 3, + "rate": 1.0 + } + }, + "operation:bottom": { + "json_valid": { + "correct": 5, + "total": 5, + "rate": 1.0 + }, + "schema_valid": { + "correct": 5, + "total": 5, + "rate": 1.0 + }, + "action_correct": { + "correct": 5, + "total": 5, + "rate": 1.0 + }, + "tool_correct": { + "correct": 5, + "total": 5, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 5, + "total": 5, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 5, + "total": 5, + "rate": 1.0 + }, + "success": { + "correct": 5, + "total": 5, + "rate": 1.0 + } + }, + "operation:lt": { + "json_valid": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "schema_valid": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "action_correct": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "tool_correct": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "success": { + "correct": 3, + "total": 3, + "rate": 1.0 + } + }, + "operation:avg": { + "json_valid": { + "correct": 5, + "total": 5, + "rate": 1.0 + }, + "schema_valid": { + "correct": 5, + "total": 5, + "rate": 1.0 + }, + "action_correct": { + "correct": 5, + "total": 5, + "rate": 1.0 + }, + "tool_correct": { + "correct": 5, + "total": 5, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 5, + "total": 5, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 5, + "total": 5, + "rate": 1.0 + }, + "success": { + "correct": 5, + "total": 5, + "rate": 1.0 + } + }, + "operation:sum": { + "json_valid": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "schema_valid": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "action_correct": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "tool_correct": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 3, + "total": 4, + "rate": 0.75 + }, + "sql_equivalent": { + "correct": 3, + "total": 4, + "rate": 0.75 + }, + "success": { + "correct": 3, + "total": 4, + "rate": 0.75 + } + }, + "language:hi": { + "json_valid": { + "correct": 48, + "total": 48, + "rate": 1.0 + }, + "schema_valid": { + "correct": 48, + "total": 48, + "rate": 1.0 + }, + "action_correct": { + "correct": 48, + "total": 48, + "rate": 1.0 + }, + "tool_correct": { + "correct": 48, + "total": 48, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 47, + "total": 48, + "rate": 0.9791666666666666 + }, + "sql_equivalent": { + "correct": 28, + "total": 29, + "rate": 0.9655172413793104 + }, + "success": { + "correct": 47, + "total": 48, + "rate": 0.9791666666666666 + } + }, + "operation:project_eq": { + "json_valid": { + "correct": 1, + "total": 1, + "rate": 1.0 + }, + "schema_valid": { + "correct": 1, + "total": 1, + "rate": 1.0 + }, + "action_correct": { + "correct": 1, + "total": 1, + "rate": 1.0 + }, + "tool_correct": { + "correct": 1, + "total": 1, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 1, + "total": 1, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 1, + "total": 1, + "rate": 1.0 + }, + "success": { + "correct": 1, + "total": 1, + "rate": 1.0 + } + }, + "operation:not_null": { + "json_valid": { + "correct": 2, + "total": 2, + "rate": 1.0 + }, + "schema_valid": { + "correct": 2, + "total": 2, + "rate": 1.0 + }, + "action_correct": { + "correct": 2, + "total": 2, + "rate": 1.0 + }, + "tool_correct": { + "correct": 2, + "total": 2, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 2, + "total": 2, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 2, + "total": 2, + "rate": 1.0 + }, + "success": { + "correct": 2, + "total": 2, + "rate": 1.0 + } + }, + "operation:or": { + "json_valid": { + "correct": 7, + "total": 7, + "rate": 1.0 + }, + "schema_valid": { + "correct": 7, + "total": 7, + "rate": 1.0 + }, + "action_correct": { + "correct": 7, + "total": 7, + "rate": 1.0 + }, + "tool_correct": { + "correct": 7, + "total": 7, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 7, + "total": 7, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 7, + "total": 7, + "rate": 1.0 + }, + "success": { + "correct": 7, + "total": 7, + "rate": 1.0 + } + }, + "operation:date_after": { + "json_valid": { + "correct": 10, + "total": 10, + "rate": 1.0 + }, + "schema_valid": { + "correct": 10, + "total": 10, + "rate": 1.0 + }, + "action_correct": { + "correct": 10, + "total": 10, + "rate": 1.0 + }, + "tool_correct": { + "correct": 10, + "total": 10, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 10, + "total": 10, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 10, + "total": 10, + "rate": 1.0 + }, + "success": { + "correct": 10, + "total": 10, + "rate": 1.0 + } + }, + "operation:top": { + "json_valid": { + "correct": 9, + "total": 9, + "rate": 1.0 + }, + "schema_valid": { + "correct": 9, + "total": 9, + "rate": 1.0 + }, + "action_correct": { + "correct": 9, + "total": 9, + "rate": 1.0 + }, + "tool_correct": { + "correct": 9, + "total": 9, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 9, + "total": 9, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 9, + "total": 9, + "rate": 1.0 + }, + "success": { + "correct": 9, + "total": 9, + "rate": 1.0 + } + }, + "operation:having": { + "json_valid": { + "correct": 5, + "total": 5, + "rate": 1.0 + }, + "schema_valid": { + "correct": 5, + "total": 5, + "rate": 1.0 + }, + "action_correct": { + "correct": 5, + "total": 5, + "rate": 1.0 + }, + "tool_correct": { + "correct": 5, + "total": 5, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 5, + "total": 5, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 5, + "total": 5, + "rate": 1.0 + }, + "success": { + "correct": 5, + "total": 5, + "rate": 1.0 + } + }, + "operation:min": { + "json_valid": { + "correct": 5, + "total": 5, + "rate": 1.0 + }, + "schema_valid": { + "correct": 5, + "total": 5, + "rate": 1.0 + }, + "action_correct": { + "correct": 5, + "total": 5, + "rate": 1.0 + }, + "tool_correct": { + "correct": 5, + "total": 5, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 5, + "total": 5, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 5, + "total": 5, + "rate": 1.0 + }, + "success": { + "correct": 5, + "total": 5, + "rate": 1.0 + } + }, + "operation:gt": { + "json_valid": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "schema_valid": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "action_correct": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "tool_correct": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "success": { + "correct": 4, + "total": 4, + "rate": 1.0 + } + }, + "backend:mysql": { + "json_valid": { + "correct": 100, + "total": 100, + "rate": 1.0 + }, + "schema_valid": { + "correct": 100, + "total": 100, + "rate": 1.0 + }, + "action_correct": { + "correct": 100, + "total": 100, + "rate": 1.0 + }, + "tool_correct": { + "correct": 100, + "total": 100, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 98, + "total": 100, + "rate": 0.98 + }, + "sql_equivalent": { + "correct": 61, + "total": 62, + "rate": 0.9838709677419355 + }, + "success": { + "correct": 98, + "total": 100, + "rate": 0.98 + } + }, + "operation:count": { + "json_valid": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "schema_valid": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "action_correct": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "tool_correct": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "success": { + "correct": 3, + "total": 3, + "rate": 1.0 + } + }, + "operation:group_sum": { + "json_valid": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "schema_valid": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "action_correct": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "tool_correct": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "success": { + "correct": 3, + "total": 3, + "rate": 1.0 + } + }, + "operation:between": { + "json_valid": { + "correct": 6, + "total": 6, + "rate": 1.0 + }, + "schema_valid": { + "correct": 6, + "total": 6, + "rate": 1.0 + }, + "action_correct": { + "correct": 6, + "total": 6, + "rate": 1.0 + }, + "tool_correct": { + "correct": 6, + "total": 6, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 6, + "total": 6, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 6, + "total": 6, + "rate": 1.0 + }, + "success": { + "correct": 6, + "total": 6, + "rate": 1.0 + } + }, + "operation:sort_asc": { + "json_valid": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "schema_valid": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "action_correct": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "tool_correct": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "success": { + "correct": 4, + "total": 4, + "rate": 1.0 + } + }, + "operation:sort_desc": { + "json_valid": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "schema_valid": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "action_correct": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "tool_correct": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "success": { + "correct": 4, + "total": 4, + "rate": 1.0 + } + }, + "operation:sql_error": { + "json_valid": { + "correct": 9, + "total": 9, + "rate": 1.0 + }, + "schema_valid": { + "correct": 9, + "total": 9, + "rate": 1.0 + }, + "action_correct": { + "correct": 9, + "total": 9, + "rate": 1.0 + }, + "tool_correct": { + "correct": 9, + "total": 9, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 8, + "total": 9, + "rate": 0.8888888888888888 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 8, + "total": 9, + "rate": 0.8888888888888888 + } + }, + "operation:all": { + "json_valid": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "schema_valid": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "action_correct": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "tool_correct": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 3, + "total": 4, + "rate": 0.75 + }, + "sql_equivalent": { + "correct": 3, + "total": 4, + "rate": 0.75 + }, + "success": { + "correct": 3, + "total": 4, + "rate": 0.75 + } + }, + "operation:and": { + "json_valid": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "schema_valid": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "action_correct": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "tool_correct": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "success": { + "correct": 4, + "total": 4, + "rate": 1.0 + } + }, + "operation:month": { + "json_valid": { + "correct": 2, + "total": 2, + "rate": 1.0 + }, + "schema_valid": { + "correct": 2, + "total": 2, + "rate": 1.0 + }, + "action_correct": { + "correct": 2, + "total": 2, + "rate": 1.0 + }, + "tool_correct": { + "correct": 2, + "total": 2, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 2, + "total": 2, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 2, + "total": 2, + "rate": 1.0 + }, + "success": { + "correct": 2, + "total": 2, + "rate": 1.0 + } + }, + "operation:join_filter": { + "json_valid": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "schema_valid": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "action_correct": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "tool_correct": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "success": { + "correct": 3, + "total": 3, + "rate": 1.0 + } + }, + "operation:eq": { + "json_valid": { + "correct": 1, + "total": 1, + "rate": 1.0 + }, + "schema_valid": { + "correct": 1, + "total": 1, + "rate": 1.0 + }, + "action_correct": { + "correct": 1, + "total": 1, + "rate": 1.0 + }, + "tool_correct": { + "correct": 1, + "total": 1, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 1, + "total": 1, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 1, + "total": 1, + "rate": 1.0 + }, + "success": { + "correct": 1, + "total": 1, + "rate": 1.0 + } + }, + "operation:list_tables": { + "json_valid": { + "correct": 12, + "total": 12, + "rate": 1.0 + }, + "schema_valid": { + "correct": 12, + "total": 12, + "rate": 1.0 + }, + "action_correct": { + "correct": 12, + "total": 12, + "rate": 1.0 + }, + "tool_correct": { + "correct": 12, + "total": 12, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 12, + "total": 12, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 12, + "total": 12, + "rate": 1.0 + } + }, + "operation:gte": { + "json_valid": { + "correct": 1, + "total": 1, + "rate": 1.0 + }, + "schema_valid": { + "correct": 1, + "total": 1, + "rate": 1.0 + }, + "action_correct": { + "correct": 1, + "total": 1, + "rate": 1.0 + }, + "tool_correct": { + "correct": 1, + "total": 1, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 1, + "total": 1, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 1, + "total": 1, + "rate": 1.0 + }, + "success": { + "correct": 1, + "total": 1, + "rate": 1.0 + } + }, + "operation:lte": { + "json_valid": { + "correct": 1, + "total": 1, + "rate": 1.0 + }, + "schema_valid": { + "correct": 1, + "total": 1, + "rate": 1.0 + }, + "action_correct": { + "correct": 1, + "total": 1, + "rate": 1.0 + }, + "tool_correct": { + "correct": 1, + "total": 1, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 1, + "total": 1, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 1, + "total": 1, + "rate": 1.0 + }, + "success": { + "correct": 1, + "total": 1, + "rate": 1.0 + } + }, + "operation:describe": { + "json_valid": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "schema_valid": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "action_correct": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "tool_correct": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 4, + "total": 4, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 4, + "total": 4, + "rate": 1.0 + } + }, + "operation:join": { + "json_valid": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "schema_valid": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "action_correct": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "tool_correct": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 3, + "total": 3, + "rate": 1.0 + }, + "success": { + "correct": 3, + "total": 3, + "rate": 1.0 + } + }, + "operation:null": { + "json_valid": { + "correct": 2, + "total": 2, + "rate": 1.0 + }, + "schema_valid": { + "correct": 2, + "total": 2, + "rate": 1.0 + }, + "action_correct": { + "correct": 2, + "total": 2, + "rate": 1.0 + }, + "tool_correct": { + "correct": 2, + "total": 2, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 2, + "total": 2, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 2, + "total": 2, + "rate": 1.0 + }, + "success": { + "correct": 2, + "total": 2, + "rate": 1.0 + } + }, + "operation:contains": { + "json_valid": { + "correct": 1, + "total": 1, + "rate": 1.0 + }, + "schema_valid": { + "correct": 1, + "total": 1, + "rate": 1.0 + }, + "action_correct": { + "correct": 1, + "total": 1, + "rate": 1.0 + }, + "tool_correct": { + "correct": 1, + "total": 1, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 1, + "total": 1, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 1, + "total": 1, + "rate": 1.0 + }, + "success": { + "correct": 1, + "total": 1, + "rate": 1.0 + } + }, + "operation:search": { + "json_valid": { + "correct": 9, + "total": 9, + "rate": 1.0 + }, + "schema_valid": { + "correct": 9, + "total": 9, + "rate": 1.0 + }, + "action_correct": { + "correct": 9, + "total": 9, + "rate": 1.0 + }, + "tool_correct": { + "correct": 9, + "total": 9, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 9, + "total": 9, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 9, + "total": 9, + "rate": 1.0 + } + }, + "operation:ticket": { + "json_valid": { + "correct": 9, + "total": 9, + "rate": 1.0 + }, + "schema_valid": { + "correct": 9, + "total": 9, + "rate": 1.0 + }, + "action_correct": { + "correct": 9, + "total": 9, + "rate": 1.0 + }, + "tool_correct": { + "correct": 9, + "total": 9, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 9, + "total": 9, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 9, + "total": 9, + "rate": 1.0 + } + }, + "operation:weather": { + "json_valid": { + "correct": 7, + "total": 7, + "rate": 1.0 + }, + "schema_valid": { + "correct": 7, + "total": 7, + "rate": 1.0 + }, + "action_correct": { + "correct": 7, + "total": 7, + "rate": 1.0 + }, + "tool_correct": { + "correct": 7, + "total": 7, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 7, + "total": 7, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 7, + "total": 7, + "rate": 1.0 + } + }, + "operation:read_file": { + "json_valid": { + "correct": 7, + "total": 7, + "rate": 1.0 + }, + "schema_valid": { + "correct": 7, + "total": 7, + "rate": 1.0 + }, + "action_correct": { + "correct": 7, + "total": 7, + "rate": 1.0 + }, + "tool_correct": { + "correct": 7, + "total": 7, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 7, + "total": 7, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 7, + "total": 7, + "rate": 1.0 + } + } + } +} \ No newline at end of file diff --git a/evaluation/gpu-final-console-metrics.json b/evaluation/gpu-final-console-metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..1763422b2328e24769e5899a12235797510e572f --- /dev/null +++ b/evaluation/gpu-final-console-metrics.json @@ -0,0 +1,83 @@ +{ + "checkpoint_sha256": "9489b5cde69c11f64b3e7031182e8248aa83790f1ad4aeefbf99a4e5e152e692", + "frozen_at_utc": "2026-09-10T14:00:36.513800+00:00", + "device": "CUDA BF16", + "source": "Captured successful remote evaluation console output; raw remote prediction files were not retrieved before SSH began refusing connections. Independent Mac evaluation is packaged separately.", + "decoding": "raw greedy; no repair, constrained decoding or teacher fallback", + "metrics": { + "test": { + "json_valid": { + "correct": 1196, + "total": 1196, + "rate": 1.0 + }, + "schema_valid": { + "correct": 1191, + "total": 1196, + "rate": 0.995819397993311 + }, + "action_correct": { + "correct": 1192, + "total": 1196, + "rate": 0.9966555183946488 + }, + "tool_correct": { + "correct": 1088, + "total": 1100, + "rate": 0.9890909090909091 + }, + "arguments_exact": { + "correct": 1014, + "total": 1100, + "rate": 0.9218181818181819 + }, + "sql_equivalent": { + "correct": 896, + "total": 912, + "rate": 0.9824561403508771 + }, + "success": { + "correct": 1106, + "total": 1196, + "rate": 0.9247491638795987 + } + }, + "manual": { + "json_valid": { + "correct": 160, + "total": 160, + "rate": 1.0 + }, + "schema_valid": { + "correct": 159, + "total": 160, + "rate": 0.99375 + }, + "action_correct": { + "correct": 159, + "total": 160, + "rate": 0.99375 + }, + "tool_correct": { + "correct": 146, + "total": 152, + "rate": 0.9605263157894737 + }, + "arguments_exact": { + "correct": 124, + "total": 152, + "rate": 0.8157894736842105 + }, + "sql_equivalent": { + "correct": 94, + "total": 112, + "rate": 0.8392857142857143 + }, + "success": { + "correct": 132, + "total": 160, + "rate": 0.825 + } + } + } +} \ No newline at end of file diff --git a/evaluation/manual-mac.jsonl b/evaluation/manual-mac.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..86af78bf138e3950e36ee42075a8ca56098bc24a --- /dev/null +++ b/evaluation/manual-mac.jsonl @@ -0,0 +1,160 @@ +{"id": "manual_2ad3e251c936135e2005_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "all", "question": "all specimens pls", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"clarify\",\"question\":\"This connection is read-only. Would you like a SELECT query instead?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": false, "tool_correct": false, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_2e2ac3c5dcf95429daaa_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "null", "question": "specimens title null records", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_0b8db53d711baed4f4ed_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "gte", "question": "specimens cost minimum 10 include same", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE cost >= 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY cost ASC LIMIT 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_2ad3e251c936135e2005_en", "language": "en", "backend": "supabase", "operation": "all", "question": "I need everything in specimens.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_efee65268e6dc2bb069e_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "project", "question": "specimens only customer_name show", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT customer_name FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT customer_name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_efee65268e6dc2bb069e_hinglish", "language": "hinglish", "backend": "supabase", "operation": "project", "question": "bas specimens ke customer_name dikha do", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT customer_name FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT customer_name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_0b8db53d711baed4f4ed_hinglish", "language": "hinglish", "backend": "supabase", "operation": "gte", "question": "specimens me cost kam se kam 10 ho", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE cost >= 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE cost >= 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_2e2ac3c5dcf95429daaa_hinglish", "language": "hinglish", "backend": "supabase", "operation": "null", "question": "specimens me jinka title NULL hai wo dikhao", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE price > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_2ad3e251c936135e2005_hinglish", "language": "hinglish", "backend": "supabase", "operation": "all", "question": "specimens ka sara data chahiye", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE display_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_d735c1ac1979ff1e8620_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sum", "question": "inspections price all add", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT SUM(price) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(price) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_0b8db53d711baed4f4ed_en", "language": "en", "backend": "supabase", "operation": "gte", "question": "Include specimens records at 10 or above in cost.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE cost >= 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE cost >= 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_2e2ac3c5dcf95429daaa_en", "language": "en", "backend": "supabase", "operation": "null", "question": "Find specimens records with no title value (NULL).", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE city = 'ApecULL';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_7dd163238936d96d0bd4_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "distinct", "question": "reservations city unique only", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT DISTINCT city FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT city FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_efee65268e6dc2bb069e_en", "language": "en", "backend": "supabase", "operation": "project", "question": "Just the customer_name values from specimens, please.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT customer_name FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT customer_name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_eb982cf071094adaca4f_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "project_eq", "question": "reservations category Delhi only name", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT name FROM reservations WHERE category = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT name FROM reservations WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_d735c1ac1979ff1e8620_en", "language": "en", "backend": "supabase", "operation": "sum", "question": "Add up price across inspections.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT SUM(price) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM inspections WHERE kind = 'Adddd';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_d735c1ac1979ff1e8620_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sum", "question": "inspections ke price ka jod batao", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT SUM(price) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(price) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_c4f25575abfd24e3cd63_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "lte", "question": "reservations cost max 1 equal also", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE cost <= 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY cost DESC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_7dd163238936d96d0bd4_hinglish", "language": "hinglish", "backend": "supabase", "operation": "distinct", "question": "reservations me city ki unique values kya hain", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT DISTINCT city FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT city FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_eb982cf071094adaca4f_hinglish", "language": "hinglish", "backend": "supabase", "operation": "project_eq", "question": "reservations me category Delhi walon ka name batao", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT name FROM reservations WHERE category = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT name FROM reservations WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_eb982cf071094adaca4f_en", "language": "en", "backend": "supabase", "operation": "project_eq", "question": "In reservations, give me name for category Delhi.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT name FROM reservations WHERE category = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT name FROM reservations WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_c4f25575abfd24e3cd63_hinglish", "language": "hinglish", "backend": "supabase", "operation": "lte", "question": "reservations me cost 1 ya usse kam ho", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE cost <= 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE cost <= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_7dd163238936d96d0bd4_en", "language": "en", "backend": "supabase", "operation": "distinct", "question": "Which different city values occur in reservations?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT DISTINCT city FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT city FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_efee65268e6dc2bb069e_hi", "language": "hi", "backend": "supabase", "operation": "project", "question": "specimens से सिर्फ customer_name दिखाना।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT customer_name FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT customer_name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_0b8db53d711baed4f4ed_hi", "language": "hi", "backend": "supabase", "operation": "gte", "question": "specimens में cost कम से कम 10 होना चाहिए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE cost >= 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE cost >= 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_d735c1ac1979ff1e8620_hi", "language": "hi", "backend": "supabase", "operation": "sum", "question": "inspections के price का जोड़ बताओ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT SUM(price) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(price) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_2ad3e251c936135e2005_hi", "language": "hi", "backend": "supabase", "operation": "all", "question": "specimens का सारा डेटा चाहिए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_c4f25575abfd24e3cd63_hi", "language": "hi", "backend": "supabase", "operation": "lte", "question": "reservations में cost 1 या उससे कम हो।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE cost <= 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE cost <= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_c4f25575abfd24e3cd63_en", "language": "en", "backend": "supabase", "operation": "lte", "question": "From reservations, include cost values up to and including 1.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE cost <= 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE cost BETWEEN 1 AND 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_eb982cf071094adaca4f_hi", "language": "hi", "backend": "supabase", "operation": "project_eq", "question": "reservations में category Delhi हो तो उनका name बताओ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT name FROM reservations WHERE category = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT name FROM reservations WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_7dd163238936d96d0bd4_hi", "language": "hi", "backend": "supabase", "operation": "distinct", "question": "reservations में city के अलग-अलग मान कौन से हैं?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT DISTINCT city FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT city FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_2e2ac3c5dcf95429daaa_hi", "language": "hi", "backend": "supabase", "operation": "null", "question": "specimens में जिनका title NULL है वे रिकॉर्ड दिखाओ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE price > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_5a8f2e207d0efe5b018c_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "list_tables", "question": "tables here list", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "manual_5a8f2e207d0efe5b018c_hinglish", "language": "hinglish", "backend": "supabase", "operation": "list_tables", "question": "yahan kaunsi tables hain", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "manual_8ebc9e868ee558a8ddfe_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "ambiguous", "question": "specimens best ones", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "manual_35406fa024e66bcb5dd7_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "lt", "question": "specimens score less 10000", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE score < 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE score < 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_82c104614df1ce6e2762_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "gt", "question": "specimens salary more than 100", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE salary > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE salary > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_82c104614df1ce6e2762_en", "language": "en", "backend": "supabase", "operation": "gt", "question": "Which specimens entries have salary above 100?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE salary > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens WHERE salary > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "manual_35406fa024e66bcb5dd7_en", "language": "en", "backend": "supabase", "operation": "lt", "question": "Find specimens records below 10000 in score.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE score < 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens WHERE score < 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "manual_b564e1a7b1e7fc6d2ab4_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "avg", "question": "specimens avg score tell", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT AVG(score) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT AVG(score) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_5a8f2e207d0efe5b018c_en", "language": "en", "backend": "supabase", "operation": "list_tables", "question": "What tables can I query here?", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "manual_8ebc9e868ee558a8ddfe_en", "language": "en", "backend": "supabase", "operation": "ambiguous", "question": "Pick the best entries in specimens.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "manual_8ebc9e868ee558a8ddfe_hinglish", "language": "hinglish", "backend": "supabase", "operation": "ambiguous", "question": "specimens me best records chun lo", "expected": {"action": "clarify", "question": "Best ka matlab kya hai? Kaunsa column aur order?"}, "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "manual_35406fa024e66bcb5dd7_hinglish", "language": "hinglish", "backend": "supabase", "operation": "lt", "question": "specimens me score 10000 se kam wale lao", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE score < 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE score < 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_82c104614df1ce6e2762_hinglish", "language": "hinglish", "backend": "supabase", "operation": "gt", "question": "specimens me salary 100 se upar wale kaun hain", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE salary > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE salary > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_b564e1a7b1e7fc6d2ab4_en", "language": "en", "backend": "supabase", "operation": "avg", "question": "What does score average out to in specimens?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT AVG(score) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT AVG(score) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "manual_b564e1a7b1e7fc6d2ab4_hinglish", "language": "hinglish", "backend": "supabase", "operation": "avg", "question": "specimens me score ka average kya hai", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT AVG(score) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT AVG(score) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_40464b00088c9159e3d0_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "count", "question": "inspections total rows how much", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT COUNT(*) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT COUNT(*) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_475287613764159ee5eb_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "group_count", "question": "reservations count per category", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_fd0bab1cc08d7298f131_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "top", "question": "exhibits top 20 by salary big first", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_40464b00088c9159e3d0_hinglish", "language": "hinglish", "backend": "supabase", "operation": "count", "question": "inspections me total kitni rows hain", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT COUNT(*) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT COUNT(*) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_40464b00088c9159e3d0_en", "language": "en", "backend": "supabase", "operation": "count", "question": "How big is inspections, measured in rows?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT COUNT(*) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT COUNT(*) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_fd0bab1cc08d7298f131_hinglish", "language": "hinglish", "backend": "supabase", "operation": "top", "question": "exhibits me sabse bade salary wali 20 rows descending dikhao", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_475287613764159ee5eb_hinglish", "language": "hinglish", "backend": "supabase", "operation": "group_count", "question": "reservations me har category ki count alag batao", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_5a8f2e207d0efe5b018c_hi", "language": "hi", "backend": "supabase", "operation": "list_tables", "question": "यहाँ कौन-कौन सी टेबल हैं?", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens WHERE kind = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "manual_b564e1a7b1e7fc6d2ab4_hi", "language": "hi", "backend": "supabase", "operation": "avg", "question": "specimens में score का औसत क्या है?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT AVG(score) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT AVG(score) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_fd0bab1cc08d7298f131_en", "language": "en", "backend": "supabase", "operation": "top", "question": "Give me 20 entries in exhibits with the largest salary, largest first.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_475287613764159ee5eb_en", "language": "en", "backend": "supabase", "operation": "group_count", "question": "Break down the number of reservations rows by category.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT COUNT(*) FROM reservations WHERE category = 'Areak downs';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_82c104614df1ce6e2762_hi", "language": "hi", "backend": "supabase", "operation": "gt", "question": "specimens में किनका salary 100 से ज़्यादा है?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE salary > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE salary > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_8ebc9e868ee558a8ddfe_hi", "language": "hi", "backend": "supabase", "operation": "ambiguous", "question": "specimens में सबसे अच्छे रिकॉर्ड चुनो।", "expected": {"action": "clarify", "question": "सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?"}, "output": "{\"action\":\"clarify\",\"question\":\"सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "manual_c4210296305105ab7472_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "ambiguous", "question": "specimens best ones", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "manual_9bf8948becf74e81a103_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "list_tables", "question": "tables here list", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "manual_9bf8948becf74e81a103_hinglish", "language": "hinglish", "backend": "mysql", "operation": "list_tables", "question": "yahan kaunsi tables hain", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_8efcf517aee64807a8e5_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "avg", "question": "specimens avg balance tell", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(balance) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_adff4eabe2639c0c128c_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "all", "question": "all inspections pls", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE description IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_35406fa024e66bcb5dd7_hi", "language": "hi", "backend": "supabase", "operation": "lt", "question": "specimens में score 10000 से कम वाले रिकॉर्ड लाओ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE score < 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE score < 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_23c0aa3a4596f995ad2a_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "count", "question": "specimens total rows how much", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_475287613764159ee5eb_hi", "language": "hi", "backend": "supabase", "operation": "group_count", "question": "reservations में हर category की गिनती अलग-अलग बताओ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_23c0aa3a4596f995ad2a_hinglish", "language": "hinglish", "backend": "mysql", "operation": "count", "question": "specimens me total kitni rows hain", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_2164b376f732a1c371ab_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "group_count", "question": "specimens count per kind", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT kind, COUNT(*) FROM specimens GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM specimens GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_40464b00088c9159e3d0_hi", "language": "hi", "backend": "supabase", "operation": "count", "question": "inspections में कुल कितनी पंक्तियाँ हैं?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT COUNT(*) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT COUNT(*) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_8efcf517aee64807a8e5_en", "language": "en", "backend": "mysql", "operation": "avg", "question": "What does balance average out to in specimens?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(balance) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_8efcf517aee64807a8e5_hinglish", "language": "hinglish", "backend": "mysql", "operation": "avg", "question": "specimens me balance ka average kya hai", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(balance) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_c4210296305105ab7472_en", "language": "en", "backend": "mysql", "operation": "ambiguous", "question": "Pick the best entries in specimens.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "manual_c4210296305105ab7472_hinglish", "language": "hinglish", "backend": "mysql", "operation": "ambiguous", "question": "specimens me best records chun lo", "expected": {"action": "clarify", "question": "Best ka matlab kya hai? Kaunsa column aur order?"}, "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "manual_23c0aa3a4596f995ad2a_en", "language": "en", "backend": "mysql", "operation": "count", "question": "How big is specimens, measured in rows?", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_d30264bbc3b08aedbc51_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sum", "question": "exhibits score all add", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT SUM(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT SUM(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_9bf8948becf74e81a103_en", "language": "en", "backend": "mysql", "operation": "list_tables", "question": "What tables can I query here?", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": false, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false, "error": "'sql' is a required property\n\nFailed validating 'required' in schema:\n {'type': 'object',\n 'properties': {'sql': {'type': 'string'}},\n 'required': ['sql'],\n 'additionalProperties': False}\n\nOn instance:\n {'schemas': ['public']}"}} +{"id": "manual_adff4eabe2639c0c128c_en", "language": "en", "backend": "mysql", "operation": "all", "question": "I need everything in inspections.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_b9c176c07809f42564a0_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "top", "question": "inspections top 5 by amount big first", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_c023a5b6f95173ae856d_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "gt", "question": "exhibits score more than 50", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM exhibits WHERE score > 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE score > 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_adff4eabe2639c0c128c_hinglish", "language": "hinglish", "backend": "mysql", "operation": "all", "question": "inspections ka sara data chahiye", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_cab55a7429427fbc5093_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "distinct", "question": "inspections department unique only", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT DISTINCT department FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_e8f90532d34c12209a83_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "gte", "question": "inspections salary minimum 1500 include same", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE salary >= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE salary >= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_c023a5b6f95173ae856d_en", "language": "en", "backend": "mysql", "operation": "gt", "question": "Which exhibits entries have score above 50?", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM exhibits WHERE score > 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE score > 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_affa5e5e14444cd82bb4_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "lt", "question": "reservations salary less 10", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE salary < 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE salary < 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_2164b376f732a1c371ab_hinglish", "language": "hinglish", "backend": "mysql", "operation": "group_count", "question": "specimens me har kind ki count alag batao", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT kind, COUNT(*) FROM specimens GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM specimens GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_d30264bbc3b08aedbc51_en", "language": "en", "backend": "mysql", "operation": "sum", "question": "Add up score across exhibits.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT SUM(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT name FROM exhibits WHERE city = 'Adddd';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false, "error": "no such column: name"}} +{"id": "manual_d30264bbc3b08aedbc51_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sum", "question": "exhibits ke score ka jod batao", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT SUM(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT SUM(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_b02f421828127e2eb96b_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "lte", "question": "exhibits price max 2 equal also", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits WHERE price <= 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits ORDER BY price DESC LIMIT 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_b9c176c07809f42564a0_hinglish", "language": "hinglish", "backend": "mysql", "operation": "top", "question": "inspections me sabse bade amount wali 5 rows descending dikhao", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_e8f90532d34c12209a83_hinglish", "language": "hinglish", "backend": "mysql", "operation": "gte", "question": "inspections me salary kam se kam 1500 ho", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE salary >= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE salary >= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_2c31add7d7b45843dec1_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "project_eq", "question": "inspections kind Pune only display_name", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT display_name FROM inspections WHERE kind = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM inspections WHERE kind = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_c023a5b6f95173ae856d_hinglish", "language": "hinglish", "backend": "mysql", "operation": "gt", "question": "exhibits me score 50 se upar wale kaun hain", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM exhibits WHERE score > 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE score > 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_2164b376f732a1c371ab_en", "language": "en", "backend": "mysql", "operation": "group_count", "question": "Break down the number of specimens rows by kind.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT kind, COUNT(*) FROM specimens GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM specimens WHERE kind = 'ureak down';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_b02f421828127e2eb96b_hinglish", "language": "hinglish", "backend": "mysql", "operation": "lte", "question": "exhibits me price 2 ya usse kam ho", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits WHERE price <= 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price <= 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_8efcf517aee64807a8e5_hi", "language": "hi", "backend": "mysql", "operation": "avg", "question": "specimens में balance का औसत क्या है?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(balance) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_cab55a7429427fbc5093_hinglish", "language": "hinglish", "backend": "mysql", "operation": "distinct", "question": "inspections me department ki unique values kya hain", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT DISTINCT department FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_b9c176c07809f42564a0_en", "language": "en", "backend": "mysql", "operation": "top", "question": "Give me 5 entries in inspections with the largest amount, largest first.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_a060620c638d2dc3759e_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "project", "question": "reservations only item_name show", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT item_name FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_affa5e5e14444cd82bb4_en", "language": "en", "backend": "mysql", "operation": "lt", "question": "Find reservations records below 10 in salary.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE salary < 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE salary < 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_affa5e5e14444cd82bb4_hinglish", "language": "hinglish", "backend": "mysql", "operation": "lt", "question": "reservations me salary 10 se kam wale lao", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE salary < 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE salary < 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_e8f90532d34c12209a83_en", "language": "en", "backend": "mysql", "operation": "gte", "question": "Include inspections records at 1500 or above in salary.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE salary >= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE salary >= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_04b2e5886e750df09a51_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "null", "question": "reservations display_name null records", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations WHERE display_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE display_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_9bf8948becf74e81a103_hi", "language": "hi", "backend": "mysql", "operation": "list_tables", "question": "यहाँ कौन-कौन सी टेबल हैं?", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE kind = 'Jaipur';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "manual_cab55a7429427fbc5093_en", "language": "en", "backend": "mysql", "operation": "distinct", "question": "Which different department values occur in inspections?", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT DISTINCT department FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_2c31add7d7b45843dec1_en", "language": "en", "backend": "mysql", "operation": "project_eq", "question": "In inspections, give me display_name for kind Pune.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT display_name FROM inspections WHERE kind = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM inspections WHERE kind = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_2c31add7d7b45843dec1_hinglish", "language": "hinglish", "backend": "mysql", "operation": "project_eq", "question": "inspections me kind Pune walon ka display_name batao", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT display_name FROM inspections WHERE kind = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM inspections WHERE kind = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_a060620c638d2dc3759e_hinglish", "language": "hinglish", "backend": "mysql", "operation": "project", "question": "bas reservations ke item_name dikha do", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT item_name FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_b02f421828127e2eb96b_en", "language": "en", "backend": "mysql", "operation": "lte", "question": "From exhibits, include price values up to and including 2.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits WHERE price <= 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price BETWEEN 2 AND 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_b02f421828127e2eb96b_hi", "language": "hi", "backend": "mysql", "operation": "lte", "question": "exhibits में price 2 या उससे कम हो।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits WHERE price <= 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price <= 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_d30264bbc3b08aedbc51_hi", "language": "hi", "backend": "mysql", "operation": "sum", "question": "exhibits के score का जोड़ बताओ।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT SUM(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT SUM(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_04b2e5886e750df09a51_hinglish", "language": "hinglish", "backend": "mysql", "operation": "null", "question": "reservations me jinka display_name NULL hai wo dikhao", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations WHERE display_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE quantity > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_a060620c638d2dc3759e_en", "language": "en", "backend": "mysql", "operation": "project", "question": "Just the item_name values from reservations, please.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT item_name FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_adff4eabe2639c0c128c_hi", "language": "hi", "backend": "mysql", "operation": "all", "question": "inspections का सारा डेटा चाहिए।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_c4210296305105ab7472_hi", "language": "hi", "backend": "mysql", "operation": "ambiguous", "question": "specimens में सबसे अच्छे रिकॉर्ड चुनो।", "expected": {"action": "clarify", "question": "सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?"}, "output": "{\"action\":\"clarify\",\"question\":\"सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "manual_23c0aa3a4596f995ad2a_hi", "language": "hi", "backend": "mysql", "operation": "count", "question": "specimens में कुल कितनी पंक्तियाँ हैं?", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_2164b376f732a1c371ab_hi", "language": "hi", "backend": "mysql", "operation": "group_count", "question": "specimens में हर kind की गिनती अलग-अलग बताओ।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT kind, COUNT(*) FROM specimens GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM specimens GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_c023a5b6f95173ae856d_hi", "language": "hi", "backend": "mysql", "operation": "gt", "question": "exhibits में किनका score 50 से ज़्यादा है?", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM exhibits WHERE score > 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE score > 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_e8f90532d34c12209a83_hi", "language": "hi", "backend": "mysql", "operation": "gte", "question": "inspections में salary कम से कम 1500 होना चाहिए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE salary >= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE salary >= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_04b2e5886e750df09a51_en", "language": "en", "backend": "mysql", "operation": "null", "question": "Find reservations records with no display_name value (NULL).", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations WHERE display_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE display_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_2c31add7d7b45843dec1_hi", "language": "hi", "backend": "mysql", "operation": "project_eq", "question": "inspections में kind Pune हो तो उनका display_name बताओ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT display_name FROM inspections WHERE kind = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM inspections WHERE kind = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_fd0bab1cc08d7298f131_hi", "language": "hi", "backend": "supabase", "operation": "top", "question": "exhibits में सबसे ज़्यादा salary वाली 20 पंक्तियाँ घटते क्रम में दिखाओ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_cab55a7429427fbc5093_hi", "language": "hi", "backend": "mysql", "operation": "distinct", "question": "inspections में department के अलग-अलग मान कौन से हैं?", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT DISTINCT department FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_a060620c638d2dc3759e_hi", "language": "hi", "backend": "mysql", "operation": "project", "question": "reservations से सिर्फ item_name दिखाना।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT item_name FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_affa5e5e14444cd82bb4_hi", "language": "hi", "backend": "mysql", "operation": "lt", "question": "reservations में salary 10 से कम वाले रिकॉर्ड लाओ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE salary < 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE salary < 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_04b2e5886e750df09a51_hi", "language": "hi", "backend": "mysql", "operation": "null", "question": "reservations में जिनका display_name NULL है वे रिकॉर्ड दिखाओ।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations WHERE display_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE quantity > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "manual_b9c176c07809f42564a0_hi", "language": "hi", "backend": "mysql", "operation": "top", "question": "inspections में सबसे ज़्यादा amount वाली 5 पंक्तियाँ घटते क्रम में दिखाओ।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "manual_88b478ce80eed61bdfb4_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "weather", "question": "Jaipur weather celsius pls", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Jaipur", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_88b478ce80eed61bdfb4_hinglish", "language": "hinglish", "backend": "supabase", "operation": "weather", "question": "Jaipur ka weather celsius me batao", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Jaipur", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_88b478ce80eed61bdfb4_en", "language": "en", "backend": "supabase", "operation": "weather", "question": "What is the weather like in Jaipur? Use celsius.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Jaipur", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_88b478ce80eed61bdfb4_hi", "language": "hi", "backend": "supabase", "operation": "weather", "question": "Jaipur का मौसम celsius में बताओ।", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Jaipur", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_4e42bbd40467b2b8c27f_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "search", "question": "docs find SQL joins", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "SQL joins"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"SQL joins\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_8a696e3c3522dd69d5ac_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "ticket", "question": "ticket TKT-4486 details", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-4486"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-4486\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_4e42bbd40467b2b8c27f_en", "language": "en", "backend": "supabase", "operation": "search", "question": "Find documentation about SQL joins.", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "SQL joins"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"SQL joins\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_f73e21e0eb10d6617995_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "read_file", "question": "/config/app.json read pls", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/config/app.json"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_8a696e3c3522dd69d5ac_hinglish", "language": "hinglish", "backend": "supabase", "operation": "ticket", "question": "ticket TKT-4486 ki details lao", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-4486"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-4486\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_4e42bbd40467b2b8c27f_hinglish", "language": "hinglish", "backend": "supabase", "operation": "search", "question": "SQL joins ke bare me docs dhundho", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "SQL joins"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"SQL joins\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_f73e21e0eb10d6617995_hinglish", "language": "hinglish", "backend": "supabase", "operation": "read_file", "question": "/config/app.json file padh kar dikhao", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/config/app.json"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_8a696e3c3522dd69d5ac_en", "language": "en", "backend": "supabase", "operation": "ticket", "question": "Fetch the details for ticket TKT-4486.", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-4486"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-4486\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_f73e21e0eb10d6617995_en", "language": "en", "backend": "supabase", "operation": "read_file", "question": "Open /config/app.json and read its text.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/config/app.json"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_8a696e3c3522dd69d5ac_hi", "language": "hi", "backend": "supabase", "operation": "ticket", "question": "टिकट TKT-4486 का विवरण लाओ।", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-4486"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-4486\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_f73e21e0eb10d6617995_hi", "language": "hi", "backend": "supabase", "operation": "read_file", "question": "/config/app.json फ़ाइल पढ़कर दिखाओ।", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/config/app.json"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_252c6053295d69a93c94_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "search", "question": "docs find database backups", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "database backups"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_5ebc52b3d20bde2076b3_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "ticket", "question": "ticket TKT-6124 details", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-6124"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6124\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_754691bd1eea25d41ccf_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "weather", "question": "Delhi weather fahrenheit pls", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_252c6053295d69a93c94_en", "language": "en", "backend": "mysql", "operation": "search", "question": "Find documentation about database backups.", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "database backups"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_4e42bbd40467b2b8c27f_hi", "language": "hi", "backend": "supabase", "operation": "search", "question": "SQL joins के बारे में दस्तावेज़ खोजो।", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "SQL joins"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"SQL joins\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_5ebc52b3d20bde2076b3_hinglish", "language": "hinglish", "backend": "mysql", "operation": "ticket", "question": "ticket TKT-6124 ki details lao", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-6124"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6124\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_754691bd1eea25d41ccf_hinglish", "language": "hinglish", "backend": "mysql", "operation": "weather", "question": "Delhi ka weather fahrenheit me batao", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_252c6053295d69a93c94_hinglish", "language": "hinglish", "backend": "mysql", "operation": "search", "question": "database backups ke bare me docs dhundho", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "database backups"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_6f3055c580b33340c854_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "read_file", "question": "/notes/setup.txt read pls", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_5ebc52b3d20bde2076b3_en", "language": "en", "backend": "mysql", "operation": "ticket", "question": "Fetch the details for ticket TKT-6124.", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-6124"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6124\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_6f3055c580b33340c854_hinglish", "language": "hinglish", "backend": "mysql", "operation": "read_file", "question": "/notes/setup.txt file padh kar dikhao", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_6f3055c580b33340c854_en", "language": "en", "backend": "mysql", "operation": "read_file", "question": "Open /notes/setup.txt and read its text.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_754691bd1eea25d41ccf_en", "language": "en", "backend": "mysql", "operation": "weather", "question": "What is the weather like in Delhi? Use fahrenheit.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_754691bd1eea25d41ccf_hi", "language": "hi", "backend": "mysql", "operation": "weather", "question": "Delhi का मौसम fahrenheit में बताओ।", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_5ebc52b3d20bde2076b3_hi", "language": "hi", "backend": "mysql", "operation": "ticket", "question": "टिकट TKT-6124 का विवरण लाओ।", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-6124"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6124\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_6f3055c580b33340c854_hi", "language": "hi", "backend": "mysql", "operation": "read_file", "question": "/notes/setup.txt फ़ाइल पढ़कर दिखाओ।", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "manual_252c6053295d69a93c94_hi", "language": "hi", "backend": "mysql", "operation": "search", "question": "database backups के बारे में दस्तावेज़ खोजो।", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "database backups"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} diff --git a/evaluation/manual-mac.summary.json b/evaluation/manual-mac.summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b0371b61e3209aafb54304eb5fe8a9a21d3288c7 --- /dev/null +++ b/evaluation/manual-mac.summary.json @@ -0,0 +1,1008 @@ +{ + "examples": 160, + "seconds": 16.877315998077393, + "decoding": "raw greedy, no repair, no teacher fallback", + "checkpoint_sha256": "9489b5cde69c11f64b3e7031182e8248aa83790f1ad4aeefbf99a4e5e152e692", + "sql_metric": "Compile against the supplied schema, then compare results on two generated SQLite fixtures after dialect adaptation; not native MySQL/PostgreSQL execution.", + "metrics": { + "json_valid": { + "correct": 160, + "total": 160, + "rate": 1.0 + }, + "schema_valid": { + "correct": 159, + "total": 160, + "rate": 0.99375 + }, + "action_correct": { + "correct": 159, + "total": 160, + "rate": 0.99375 + }, + "tool_correct": { + "correct": 146, + "total": 152, + "rate": 0.9605263157894737 + }, + "arguments_exact": { + "correct": 124, + "total": 152, + "rate": 0.8157894736842105 + }, + "sql_equivalent": { + "correct": 94, + "total": 112, + "rate": 0.8392857142857143 + }, + "success": { + "correct": 132, + "total": 160, + "rate": 0.825 + } + }, + "groups": { + "language:noisy_en": { + "json_valid": { + "correct": 40, + "total": 40, + "rate": 1.0 + }, + "schema_valid": { + "correct": 40, + "total": 40, + "rate": 1.0 + }, + "action_correct": { + "correct": 39, + "total": 40, + "rate": 0.975 + }, + "tool_correct": { + "correct": 35, + "total": 38, + "rate": 0.9210526315789473 + }, + "arguments_exact": { + "correct": 31, + "total": 38, + "rate": 0.8157894736842105 + }, + "sql_equivalent": { + "correct": 23, + "total": 28, + "rate": 0.8214285714285714 + }, + "success": { + "correct": 33, + "total": 40, + "rate": 0.825 + } + }, + "backend:supabase": { + "json_valid": { + "correct": 80, + "total": 80, + "rate": 1.0 + }, + "schema_valid": { + "correct": 80, + "total": 80, + "rate": 1.0 + }, + "action_correct": { + "correct": 79, + "total": 80, + "rate": 0.9875 + }, + "tool_correct": { + "correct": 73, + "total": 76, + "rate": 0.9605263157894737 + }, + "arguments_exact": { + "correct": 59, + "total": 76, + "rate": 0.7763157894736842 + }, + "sql_equivalent": { + "correct": 46, + "total": 56, + "rate": 0.8214285714285714 + }, + "success": { + "correct": 63, + "total": 80, + "rate": 0.7875 + } + }, + "operation:all": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 7, + "total": 8, + "rate": 0.875 + }, + "tool_correct": { + "correct": 7, + "total": 8, + "rate": 0.875 + }, + "arguments_exact": { + "correct": 5, + "total": 8, + "rate": 0.625 + }, + "sql_equivalent": { + "correct": 5, + "total": 8, + "rate": 0.625 + }, + "success": { + "correct": 5, + "total": 8, + "rate": 0.625 + } + }, + "operation:null": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 2, + "total": 8, + "rate": 0.25 + }, + "sql_equivalent": { + "correct": 2, + "total": 8, + "rate": 0.25 + }, + "success": { + "correct": 2, + "total": 8, + "rate": 0.25 + } + }, + "operation:gte": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 7, + "total": 8, + "rate": 0.875 + }, + "sql_equivalent": { + "correct": 7, + "total": 8, + "rate": 0.875 + }, + "success": { + "correct": 7, + "total": 8, + "rate": 0.875 + } + }, + "language:en": { + "json_valid": { + "correct": 40, + "total": 40, + "rate": 1.0 + }, + "schema_valid": { + "correct": 39, + "total": 40, + "rate": 0.975 + }, + "action_correct": { + "correct": 40, + "total": 40, + "rate": 1.0 + }, + "tool_correct": { + "correct": 37, + "total": 38, + "rate": 0.9736842105263158 + }, + "arguments_exact": { + "correct": 25, + "total": 38, + "rate": 0.6578947368421053 + }, + "sql_equivalent": { + "correct": 20, + "total": 28, + "rate": 0.7142857142857143 + }, + "success": { + "correct": 27, + "total": 40, + "rate": 0.675 + } + }, + "operation:project": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "success": { + "correct": 8, + "total": 8, + "rate": 1.0 + } + }, + "language:hinglish": { + "json_valid": { + "correct": 40, + "total": 40, + "rate": 1.0 + }, + "schema_valid": { + "correct": 40, + "total": 40, + "rate": 1.0 + }, + "action_correct": { + "correct": 40, + "total": 40, + "rate": 1.0 + }, + "tool_correct": { + "correct": 38, + "total": 38, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 34, + "total": 38, + "rate": 0.8947368421052632 + }, + "sql_equivalent": { + "correct": 25, + "total": 28, + "rate": 0.8928571428571429 + }, + "success": { + "correct": 36, + "total": 40, + "rate": 0.9 + } + }, + "operation:sum": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 6, + "total": 8, + "rate": 0.75 + }, + "sql_equivalent": { + "correct": 6, + "total": 8, + "rate": 0.75 + }, + "success": { + "correct": 6, + "total": 8, + "rate": 0.75 + } + }, + "operation:distinct": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "success": { + "correct": 8, + "total": 8, + "rate": 1.0 + } + }, + "operation:project_eq": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "success": { + "correct": 8, + "total": 8, + "rate": 1.0 + } + }, + "operation:lte": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 4, + "total": 8, + "rate": 0.5 + }, + "sql_equivalent": { + "correct": 4, + "total": 8, + "rate": 0.5 + }, + "success": { + "correct": 4, + "total": 8, + "rate": 0.5 + } + }, + "language:hi": { + "json_valid": { + "correct": 40, + "total": 40, + "rate": 1.0 + }, + "schema_valid": { + "correct": 40, + "total": 40, + "rate": 1.0 + }, + "action_correct": { + "correct": 40, + "total": 40, + "rate": 1.0 + }, + "tool_correct": { + "correct": 36, + "total": 38, + "rate": 0.9473684210526315 + }, + "arguments_exact": { + "correct": 34, + "total": 38, + "rate": 0.8947368421052632 + }, + "sql_equivalent": { + "correct": 26, + "total": 28, + "rate": 0.9285714285714286 + }, + "success": { + "correct": 36, + "total": 40, + "rate": 0.9 + } + }, + "operation:list_tables": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 7, + "total": 8, + "rate": 0.875 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 3, + "total": 8, + "rate": 0.375 + }, + "arguments_exact": { + "correct": 1, + "total": 8, + "rate": 0.125 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 1, + "total": 8, + "rate": 0.125 + } + }, + "operation:ambiguous": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "arguments_exact": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 8, + "total": 8, + "rate": 1.0 + } + }, + "operation:lt": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 7, + "total": 8, + "rate": 0.875 + }, + "sql_equivalent": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "success": { + "correct": 7, + "total": 8, + "rate": 0.875 + } + }, + "operation:gt": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 7, + "total": 8, + "rate": 0.875 + }, + "sql_equivalent": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "success": { + "correct": 7, + "total": 8, + "rate": 0.875 + } + }, + "operation:avg": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 7, + "total": 8, + "rate": 0.875 + }, + "sql_equivalent": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "success": { + "correct": 7, + "total": 8, + "rate": 0.875 + } + }, + "operation:count": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "success": { + "correct": 8, + "total": 8, + "rate": 1.0 + } + }, + "operation:group_count": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 6, + "total": 8, + "rate": 0.75 + }, + "sql_equivalent": { + "correct": 6, + "total": 8, + "rate": 0.75 + }, + "success": { + "correct": 6, + "total": 8, + "rate": 0.75 + } + }, + "operation:top": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "success": { + "correct": 8, + "total": 8, + "rate": 1.0 + } + }, + "backend:mysql": { + "json_valid": { + "correct": 80, + "total": 80, + "rate": 1.0 + }, + "schema_valid": { + "correct": 79, + "total": 80, + "rate": 0.9875 + }, + "action_correct": { + "correct": 80, + "total": 80, + "rate": 1.0 + }, + "tool_correct": { + "correct": 73, + "total": 76, + "rate": 0.9605263157894737 + }, + "arguments_exact": { + "correct": 65, + "total": 76, + "rate": 0.8552631578947368 + }, + "sql_equivalent": { + "correct": 48, + "total": 56, + "rate": 0.8571428571428571 + }, + "success": { + "correct": 69, + "total": 80, + "rate": 0.8625 + } + }, + "operation:weather": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 8, + "total": 8, + "rate": 1.0 + } + }, + "operation:search": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 8, + "total": 8, + "rate": 1.0 + } + }, + "operation:ticket": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 8, + "total": 8, + "rate": 1.0 + } + }, + "operation:read_file": { + "json_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "schema_valid": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "action_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "tool_correct": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 8, + "total": 8, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 8, + "total": 8, + "rate": 1.0 + } + } + } +} \ No newline at end of file diff --git a/evaluation/test-mac.jsonl b/evaluation/test-mac.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..beb77ab83f5701cdbedf3cbc352b671cca89b4bc --- /dev/null +++ b/evaluation/test-mac.jsonl @@ -0,0 +1,1196 @@ +{"id": "8234746878d4297cb07a_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "missing_schema", "question": "Since no schema is provided by me, find tables before writing sql for inspections.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "8234746878d4297cb07a_en", "language": "en", "backend": "supabase", "operation": "missing_schema", "question": "Given that I have not provided a database schema, find the available tables before writing SQL for inspections.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "8234746878d4297cb07a_hinglish", "language": "hinglish", "backend": "supabase", "operation": "missing_schema", "question": "Kripya inspections ke liye SQL likhne se pehle available tables ka pata lagao, kyunki maine database schema nahi diya hai.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "797213190eddac109b96_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sql_error", "question": "The last one failed because of a unknown column. Please inspect inspections schema before retrying.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "41f5fbcefb9d550068a0_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "missing_schema", "question": "Since no schema is provided by me, find tables before writing sql for specimens.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "4b73bea165fb5f9555f3_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sql_error", "question": "The last one failed because of a unknown column. Please inspect inspections schema before retrying.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "0a2ec84ba0e594d94b9e_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sql_error", "question": "The last one failed because of a unknown column. Please inspect reservations schema before retrying.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "44135783de9bd091bfe5_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "missing_schema", "question": "Since no schema is provided by me, find tables before writing sql for inspections.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_inspections", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_inspections\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "6db4ac493194a52cbd3a_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "missing_schema", "question": "Since no schema is provided by me, find tables before writing sql for reservations.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_reservations", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_reservations\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "797213190eddac109b96_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sql_error", "question": "Pichla query fail raha kyunki ek column unknown tha. Dobara try karne se pehle inspections schema verify karo.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "4b73bea165fb5f9555f3_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sql_error", "question": "Pichla query fail raha kyunki ek column unknown tha. Dobara try karne se pehle inspections schema verify karo.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "0a2ec84ba0e594d94b9e_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sql_error", "question": "Pichla query fail raha kyunki ek column unknown tha. Dobara try karne se pehle reservations schema verify karo.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "797213190eddac109b96_en", "language": "en", "backend": "supabase", "operation": "sql_error", "question": "An unknown column caused the last query to fail; check inspections schema before your next attempt.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "4b73bea165fb5f9555f3_en", "language": "en", "backend": "supabase", "operation": "sql_error", "question": "An unknown column caused the last query to fail; check inspections schema before your next attempt.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "0a2ec84ba0e594d94b9e_en", "language": "en", "backend": "supabase", "operation": "sql_error", "question": "An unknown column caused the last query to fail; check reservations schema before your next attempt.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "41f5fbcefb9d550068a0_en", "language": "en", "backend": "supabase", "operation": "missing_schema", "question": "Given that I have not provided a database schema, find the available tables before writing SQL for specimens.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "41f5fbcefb9d550068a0_hinglish", "language": "hinglish", "backend": "supabase", "operation": "missing_schema", "question": "Kripya specimens ke liye SQL likhne se pehle available tables ka pata lagao, kyunki maine database schema nahi diya hai.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "44135783de9bd091bfe5_en", "language": "en", "backend": "supabase", "operation": "missing_schema", "question": "Given that I have not provided a database schema, find the available tables before writing SQL for inspections.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_inspections", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_inspections\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "6db4ac493194a52cbd3a_en", "language": "en", "backend": "supabase", "operation": "missing_schema", "question": "Given that I have not provided a database schema, find the available tables before writing SQL for reservations.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_reservations", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_reservations\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "44135783de9bd091bfe5_hinglish", "language": "hinglish", "backend": "supabase", "operation": "missing_schema", "question": "Kripya inspections ke liye SQL likhne se pehle available tables ka pata lagao, kyunki maine database schema nahi diya hai.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_inspections", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_inspections\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "6db4ac493194a52cbd3a_hinglish", "language": "hinglish", "backend": "supabase", "operation": "missing_schema", "question": "Kripya reservations ke liye SQL likhne se pehle available tables ka pata lagao, kyunki maine database schema nahi diya hai.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_reservations", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_reservations\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "84b16996198346dc9882_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "missing_schema", "question": "Since no schema is provided by me, find tables before writing sql for exhibits.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "5cb3a450bcdc957b7ba5_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "missing_schema", "question": "Since no schema is provided by me, find tables before writing sql for specimens.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "578a0a8a98b724c87050_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "list_tables", "question": "tell me the tables in this database", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "5689f342b9bf3175e2fe_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "all", "question": "fetch all rows from specimens", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "578a0a8a98b724c87050_en", "language": "en", "backend": "supabase", "operation": "list_tables", "question": "Display the schema tables in this database.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "578a0a8a98b724c87050_hinglish", "language": "hinglish", "backend": "supabase", "operation": "list_tables", "question": "Is database ki tables ka detail do.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "32876ff450ca027d658e_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "count", "question": "how many lines in specimens?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5689f342b9bf3175e2fe_en", "language": "en", "backend": "supabase", "operation": "all", "question": "Get all rows from specimens.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2ad3e251c936135e2005_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "all", "question": "fetch all rows from specimens", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "32876ff450ca027d658e_hinglish", "language": "hinglish", "backend": "supabase", "operation": "count", "question": "Batao specimens mein kitni entries hain.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "84b16996198346dc9882_en", "language": "en", "backend": "mysql", "operation": "missing_schema", "question": "Given that I have not provided a database schema, find the available tables before writing SQL for exhibits.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "2ad3e251c936135e2005_en", "language": "en", "backend": "supabase", "operation": "all", "question": "Get all rows from specimens.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "84b16996198346dc9882_hinglish", "language": "hinglish", "backend": "mysql", "operation": "missing_schema", "question": "Kripya exhibits ke liye SQL likhne se pehle available tables ka pata lagao, kyunki maine database schema nahi diya hai.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "5cb3a450bcdc957b7ba5_en", "language": "en", "backend": "mysql", "operation": "missing_schema", "question": "Given that I have not provided a database schema, find the available tables before writing SQL for specimens.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "772353a7da7eebcf3244_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "avg", "question": "determine the mean quantity in specimens", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(quantity) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(quantity) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e4a81e1122226c7b26d9_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "describe", "question": "Look up the columns and types for specimens.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "5689f342b9bf3175e2fe_hinglish", "language": "hinglish", "backend": "supabase", "operation": "all", "question": "specimens ke saare records get karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "efee65268e6dc2bb069e_hinglish", "language": "hinglish", "backend": "supabase", "operation": "project", "question": "bas specimens ka customer_name do please", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT customer_name FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT customer_name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0b8db53d711baed4f4ed_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "gte", "question": "output records from specimens where cost is not below 10", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE cost >= 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE cost >= 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5cb3a450bcdc957b7ba5_hinglish", "language": "hinglish", "backend": "mysql", "operation": "missing_schema", "question": "Kripya specimens ke liye SQL likhne se pehle available tables ka pata lagao, kyunki maine database schema nahi diya hai.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "0ffff93d2f774a97e3f3_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "min", "question": "What is the floor price in specimens?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MIN(price) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(price) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "efee65268e6dc2bb069e_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "project", "question": "display customer_name from specimens and nothing else", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT customer_name FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT customer_name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2e2ac3c5dcf95429daaa_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "null", "question": "Show me the rows from specimens where title is null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2e2ac3c5dcf95429daaa_hinglish", "language": "hinglish", "backend": "supabase", "operation": "null", "question": "specimens se title null wali rows select karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2ad3e251c936135e2005_hinglish", "language": "hinglish", "backend": "supabase", "operation": "all", "question": "specimens ke saare records get karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2984a6707d4e07f81695_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "write", "question": "Remove all records from exhibits now.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"clarify\",\"question\":\"This connection is read-only. Would you like a SELECT query instead?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "772353a7da7eebcf3244_en", "language": "en", "backend": "supabase", "operation": "avg", "question": "Return the average quantity in specimens.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(quantity) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(quantity) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ad0842eb95739a0c88ec_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "avg", "question": "determine the mean balance in inspections", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(balance) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e4a81e1122226c7b26d9_en", "language": "en", "backend": "supabase", "operation": "describe", "question": "Reveal the column structure and types for specimens.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "7a843b154fbd48aac50b_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "all", "question": "fetch all rows from inspections", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4f5c410b4b3379c58420_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "or", "question": "Display rows from specimens where category is pending or North.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE category IN ('pending', 'North');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE category IN ('pending', 'North');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0b8db53d711baed4f4ed_en", "language": "en", "backend": "supabase", "operation": "gte", "question": "Show the data from specimens where cost is 10 or above.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE cost >= 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE cost >= 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "80ed46f6e275ed01c6c1_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "count_eq", "question": "how many exhibits rows where category is Pune", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM exhibits WHERE category = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM exhibits WHERE category = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "fea44ee93f1cea0883dd_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "count_eq", "question": "how many exhibits rows where category is Delhi", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM exhibits WHERE category = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM exhibits WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e3d12f1bf51bbf54a331_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "contains", "question": "List specimens rows where description equals North, ignore case differences.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE LOWER(description) LIKE LOWER('%North%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE LOWER(description) LIKE LOWER('%North%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "96a74866d51ea7899cd8_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "gte", "question": "output records from specimens where score is not below 20", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE score >= 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE score >= 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "772353a7da7eebcf3244_hinglish", "language": "hinglish", "backend": "supabase", "operation": "avg", "question": "specimens me quantity ka average return karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(quantity) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(quantity) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1395b37e5ba28fa3c134_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "min", "question": "What is the floor amount in inspections?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MIN(amount) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(amount) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e4a81e1122226c7b26d9_hinglish", "language": "hinglish", "backend": "supabase", "operation": "describe", "question": "Mujhe specimens ke columns aur types ke baare mein info do.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "7a843b154fbd48aac50b_en", "language": "en", "backend": "supabase", "operation": "all", "question": "Get all rows from inspections.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2e2ac3c5dcf95429daaa_en", "language": "en", "backend": "supabase", "operation": "null", "question": "I want to view the rows in specimens where title is null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d735c1ac1979ff1e8620_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sum", "question": "i want the total price of inspections", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT SUM(price) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(price) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "28a7ce61b1233483bf73_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sort_desc", "question": "retrieve all records from specimens sorted by salary high to low", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens ORDER BY salary DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c15c59356c7863f0b7e9_en", "language": "en", "backend": "supabase", "operation": "eq", "question": "Bring up all rows from specimens where city has the value Books.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE city = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE city = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c15c59356c7863f0b7e9_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "eq", "question": "Get the full list of rows in specimens where city is Books.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE city = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE city = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "358364da33322bbf17e2_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "project_eq", "question": "give me title from exhibits where city is Pune", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT title FROM exhibits WHERE city = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT title FROM exhibits WHERE city = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ad0842eb95739a0c88ec_en", "language": "en", "backend": "supabase", "operation": "avg", "question": "Return the average balance in inspections.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(balance) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ad0842eb95739a0c88ec_hinglish", "language": "hinglish", "backend": "supabase", "operation": "avg", "question": "inspections me balance ka average return karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(balance) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "65666802d11552ba1fc2_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "write", "question": "Remove all records from inspections now.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"clarify\",\"question\":\"This connection is read-only. Would you like a SELECT query instead?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "32876ff450ca027d658e_en", "language": "en", "backend": "supabase", "operation": "count", "question": "What is the volume of data rows in specimens?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0ffff93d2f774a97e3f3_en", "language": "en", "backend": "supabase", "operation": "min", "question": "What is the baseline price value in specimens?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MIN(price) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(price) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "0ffff93d2f774a97e3f3_hinglish", "language": "hinglish", "backend": "supabase", "operation": "min", "question": "specimens mein price ka minimum reading kya hai?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MIN(price) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(price) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4f5c410b4b3379c58420_en", "language": "en", "backend": "supabase", "operation": "or", "question": "Bring up rows from specimens where category is either pending or North.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE category IN ('pending', 'North');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE category IN ('pending', 'North');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4f5c410b4b3379c58420_hinglish", "language": "hinglish", "backend": "supabase", "operation": "or", "question": "specimens mein category pending ya North wali rows dikhao na.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE category IN ('pending', 'North');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE category IN ('pending', 'North');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "efee65268e6dc2bb069e_en", "language": "en", "backend": "supabase", "operation": "project", "question": "Display customer_name from specimens as the sole result.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT customer_name FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT customer_name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0b8db53d711baed4f4ed_hinglish", "language": "hinglish", "backend": "supabase", "operation": "gte", "question": "specimens se rows dikhao jahan cost 10 ya usse bada hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE cost >= 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE cost >= 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "84b620e6d482597eea06_hinglish", "language": "hinglish", "backend": "supabase", "operation": "project", "question": "bas exhibits ka display_name do please", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT display_name FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2984a6707d4e07f81695_en", "language": "en", "backend": "supabase", "operation": "write", "question": "Expunge every row from exhibits.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"clarify\",\"question\":\"This connection is read-only. Would you like a SELECT query instead?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "2984a6707d4e07f81695_hinglish", "language": "hinglish", "backend": "supabase", "operation": "write", "question": "exhibits ka pura content hatwa do.", "expected": {"action": "clarify", "question": "Yeh connection read-only hai. Kya SELECT query chahiye?"}, "output": "{\"action\":\"clarify\",\"question\":\"Yeh connection read-only hai. Kya SELECT query chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "96a74866d51ea7899cd8_en", "language": "en", "backend": "supabase", "operation": "gte", "question": "Show the data from specimens where score is 20 or above.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE score >= 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE score >= 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8234746878d4297cb07a_hi", "language": "hi", "backend": "supabase", "operation": "missing_schema", "question": "कृपया inspections के लिए SQL लिखने से पहले उपलब्ध तालिकाओं का पता लगाएं, क्योंकि मैंने डेटाबेस स्कीमा नहीं दी है।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "358364da33322bbf17e2_en", "language": "en", "backend": "supabase", "operation": "project_eq", "question": "List the title from exhibits where city is Pune.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT title FROM exhibits WHERE city = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT title FROM exhibits WHERE city = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1395b37e5ba28fa3c134_hinglish", "language": "hinglish", "backend": "supabase", "operation": "min", "question": "inspections mein amount ka minimum reading kya hai?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MIN(amount) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(amount) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7a843b154fbd48aac50b_hinglish", "language": "hinglish", "backend": "supabase", "operation": "all", "question": "inspections ke saare records get karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "80ed46f6e275ed01c6c1_en", "language": "en", "backend": "supabase", "operation": "count_eq", "question": "Provide the row count for exhibits where category is Pune.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM exhibits WHERE category = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM exhibits WHERE category = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "80ed46f6e275ed01c6c1_hinglish", "language": "hinglish", "backend": "supabase", "operation": "count_eq", "question": "Please count the rows in exhibits where category is Pune.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM exhibits WHERE category = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM exhibits WHERE category = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d735c1ac1979ff1e8620_en", "language": "en", "backend": "supabase", "operation": "sum", "question": "Output the total price for the inspections entity.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT SUM(price) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(price) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "28a7ce61b1233483bf73_en", "language": "en", "backend": "supabase", "operation": "sort_desc", "question": "Display the entire table specimens sorted by salary from highest to lowest.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens ORDER BY salary DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "84b620e6d482597eea06_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "project", "question": "display display_name from exhibits and nothing else", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT display_name FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bcd7a532a06274ddddee_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "or", "question": "Display rows from exhibits where status is pending or South.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE status IN ('pending', 'South');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE status IN ('pending', 'South');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "fea44ee93f1cea0883dd_en", "language": "en", "backend": "supabase", "operation": "count_eq", "question": "Provide the row count for exhibits where category is Delhi.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM exhibits WHERE category = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM exhibits WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "fea44ee93f1cea0883dd_hinglish", "language": "hinglish", "backend": "supabase", "operation": "count_eq", "question": "Please count the rows in exhibits where category is Delhi.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM exhibits WHERE category = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM exhibits WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8a456280df4a711c3cda_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "year", "question": "Fetch inspections entries where created_at year is 2024.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE EXTRACT(YEAR FROM created_at) = 2024;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE EXTRACT(YEAR FROM created_at) = 2024;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c15c59356c7863f0b7e9_hinglish", "language": "hinglish", "backend": "supabase", "operation": "eq", "question": "specimens se wo saari rows lao jahan city ka value Books hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE city = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE city = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a7afc1ab21c25c700603_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "or", "question": "Display rows from exhibits where status is Mumbai or Electronics.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE status IN ('Mumbai', 'Electronics');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE status IN ('Mumbai', 'Electronics');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1395b37e5ba28fa3c134_en", "language": "en", "backend": "supabase", "operation": "min", "question": "What is the baseline amount value in inspections?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MIN(amount) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(amount) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "71349134f8ae16809501_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sort_desc", "question": "retrieve all records from exhibits sorted by salary high to low", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits ORDER BY salary DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ac83eae59947f137269f_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "write", "question": "Remove all records from reservations now.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"clarify\",\"question\":\"This connection is read-only. Would you like a SELECT query instead?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "28a7ce61b1233483bf73_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sort_desc", "question": "specimens ki poori table salary highest se lowest order mein dikhao.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens ORDER BY salary DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1a1af4deb475f0224a59_en", "language": "en", "backend": "supabase", "operation": "bottom", "question": "Present the 10 minimal price entries from inspections in ascending order.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections ORDER BY price ASC LIMIT 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY price ASC LIMIT 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1a1af4deb475f0224a59_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "bottom", "question": "print the 10 minimal price rows in inspections from lowest up", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections ORDER BY price ASC LIMIT 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY price ASC LIMIT 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bcd7a532a06274ddddee_hinglish", "language": "hinglish", "backend": "supabase", "operation": "or", "question": "exhibits mein status pending ya South wali rows dikhao na.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE status IN ('pending', 'South');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE status IN ('pending', 'South');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7dd163238936d96d0bd4_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "distinct", "question": "i need the distinct city values from reservations", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT DISTINCT city FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT city FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8a456280df4a711c3cda_en", "language": "en", "backend": "supabase", "operation": "year", "question": "Show me the inspections rows for 2024 using created_at.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE EXTRACT(YEAR FROM created_at) = 2024;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE EXTRACT(YEAR FROM created_at) = 2024;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8a456280df4a711c3cda_hinglish", "language": "hinglish", "backend": "supabase", "operation": "year", "question": "inspections mein created_at mein 2024 wale records dikhao.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE EXTRACT(YEAR FROM created_at) = 2024;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE EXTRACT(YEAR FROM created_at) = 2024;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "96a74866d51ea7899cd8_hinglish", "language": "hinglish", "backend": "supabase", "operation": "gte", "question": "specimens se rows dikhao jahan score 20 ya usse bada hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE score >= 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE score >= 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4745f53221e07d4f5ae1_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "contains", "question": "List specimens rows where display_name equals Pune, ignore case differences.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE LOWER(display_name) LIKE LOWER('%Pune%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE LOWER(display_name) LIKE LOWER('%Pune%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "358364da33322bbf17e2_hinglish", "language": "hinglish", "backend": "supabase", "operation": "project_eq", "question": "exhibits se title ki list do jahan city Pune hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT title FROM exhibits WHERE city = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT title FROM exhibits WHERE city = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a7afc1ab21c25c700603_hinglish", "language": "hinglish", "backend": "supabase", "operation": "or", "question": "exhibits mein status Mumbai ya Electronics wali rows dikhao na.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE status IN ('Mumbai', 'Electronics');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE status IN ('Mumbai', 'Electronics');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6b9bacf4d9aa380af183_en", "language": "en", "backend": "supabase", "operation": "ambiguous", "question": "Print the best available records in reservations.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "6b9bacf4d9aa380af183_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "ambiguous", "question": "Give me the top records in reservations.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "6b9bacf4d9aa380af183_hinglish", "language": "hinglish", "backend": "supabase", "operation": "ambiguous", "question": "reservations mein available best records print karo.", "expected": {"action": "clarify", "question": "Best ka matlab kya hai? Kaunsa column aur order?"}, "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "65666802d11552ba1fc2_en", "language": "en", "backend": "supabase", "operation": "write", "question": "Expunge every row from inspections.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"clarify\",\"question\":\"This connection is read-only. Would you like a SELECT query instead?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "65666802d11552ba1fc2_hinglish", "language": "hinglish", "backend": "supabase", "operation": "write", "question": "inspections ka pura content hatwa do.", "expected": {"action": "clarify", "question": "Yeh connection read-only hai. Kya SELECT query chahiye?"}, "output": "{\"action\":\"clarify\",\"question\":\"Yeh connection read-only hai. Kya SELECT query chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "e91a7141af66518c19a2_en", "language": "en", "backend": "supabase", "operation": "max", "question": "What is the largest number for cost in inspections?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MAX(cost) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(cost) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e91a7141af66518c19a2_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "max", "question": "what is the upper bound of cost in inspections?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MAX(cost) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(cost) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "dd6cdefd15edcdcddb76_en", "language": "en", "backend": "supabase", "operation": "bottom", "question": "Present the 5 minimal salary entries from exhibits in ascending order.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits ORDER BY salary ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY salary ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "dd6cdefd15edcdcddb76_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "bottom", "question": "print the 5 minimal salary rows in exhibits from lowest up", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits ORDER BY salary ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY salary ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d735c1ac1979ff1e8620_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sum", "question": "inspections entity ke liye price ka total output do.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT SUM(price) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(price) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c8fd1555fd64b08f1241_en", "language": "en", "backend": "supabase", "operation": "eq", "question": "Bring up all rows from inspections where city has the value Electronics.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE city = 'Electronics';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE city = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c8fd1555fd64b08f1241_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "eq", "question": "Get the full list of rows in inspections where city is Electronics.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE city = 'Electronics';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE city = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "eb982cf071094adaca4f_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "project_eq", "question": "give me name from reservations where category is Delhi", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT name FROM reservations WHERE category = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT name FROM reservations WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bcd7a532a06274ddddee_en", "language": "en", "backend": "supabase", "operation": "or", "question": "Bring up rows from exhibits where status is either pending or South.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE status IN ('pending', 'South');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE status IN ('pending', 'South');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e3d12f1bf51bbf54a331_hinglish", "language": "hinglish", "backend": "supabase", "operation": "contains", "question": "Output karo specimens mein description mein North match hone wali rows, case insensitive raho.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE LOWER(description) LIKE LOWER('%North%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE LOWER(description) LIKE LOWER('%North%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "772353a7da7eebcf3244_hi", "language": "hi", "backend": "supabase", "operation": "avg", "question": "specimens में quantity का औसत लौटाएँ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(quantity) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(quantity) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a7afc1ab21c25c700603_en", "language": "en", "backend": "supabase", "operation": "or", "question": "Bring up rows from exhibits where status is either Mumbai or Electronics.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE status IN ('Mumbai', 'Electronics');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE status IN ('Mumbai', 'Electronics');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "71349134f8ae16809501_en", "language": "en", "backend": "supabase", "operation": "sort_desc", "question": "Display the entire table exhibits sorted by salary from highest to lowest.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits ORDER BY salary DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "71349134f8ae16809501_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sort_desc", "question": "exhibits ki poori table salary highest se lowest order mein dikhao.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits ORDER BY salary DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e91a7141af66518c19a2_hinglish", "language": "hinglish", "backend": "supabase", "operation": "max", "question": "inspections mein cost ka sabse bada number kya hai?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MAX(cost) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(cost) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ac83eae59947f137269f_hinglish", "language": "hinglish", "backend": "supabase", "operation": "write", "question": "reservations ka pura content hatwa do.", "expected": {"action": "clarify", "question": "Yeh connection read-only hai. Kya SELECT query chahiye?"}, "output": "{\"action\":\"clarify\",\"question\":\"Yeh connection read-only hai. Kya SELECT query chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "c8fd1555fd64b08f1241_hinglish", "language": "hinglish", "backend": "supabase", "operation": "eq", "question": "inspections se wo saari rows lao jahan city ka value Electronics hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE city = 'Electronics';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE city = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "eb982cf071094adaca4f_en", "language": "en", "backend": "supabase", "operation": "project_eq", "question": "List the name from reservations where category is Delhi.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT name FROM reservations WHERE category = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT name FROM reservations WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "84b620e6d482597eea06_en", "language": "en", "backend": "supabase", "operation": "project", "question": "Display display_name from exhibits as the sole result.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT display_name FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "92e36c8a6ca6e7aa2298_en", "language": "en", "backend": "supabase", "operation": "group_count", "question": "Find the count of rows per category in reservations.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "92e36c8a6ca6e7aa2298_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "group_count", "question": "give me the count for each category rows in reservations", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7d18561ec942f18f1299_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "count_eq", "question": "how many reservations rows where status is O'Reilly", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM reservations WHERE status = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM reservations WHERE status = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ad0842eb95739a0c88ec_hi", "language": "hi", "backend": "supabase", "operation": "avg", "question": "inspections में balance का औसत लौटाएँ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(balance) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4dac1dcf2ef14bfee2d2_en", "language": "en", "backend": "supabase", "operation": "group_count", "question": "Find the count of rows per category in reservations.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4dac1dcf2ef14bfee2d2_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "group_count", "question": "give me the count for each category rows in reservations", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ac83eae59947f137269f_en", "language": "en", "backend": "supabase", "operation": "write", "question": "Expunge every row from reservations.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"clarify\",\"question\":\"This connection is read-only. Would you like a SELECT query instead?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "efee65268e6dc2bb069e_hi", "language": "hi", "backend": "supabase", "operation": "project", "question": "सिर्फ specimens का customer_name दिखाइए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT customer_name FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT customer_name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "eb982cf071094adaca4f_hinglish", "language": "hinglish", "backend": "supabase", "operation": "project_eq", "question": "reservations se name ki list do jahan category Delhi hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT name FROM reservations WHERE category = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT name FROM reservations WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b30330bb2c17b32c6883_en", "language": "en", "backend": "supabase", "operation": "sort_asc", "question": "View all records in exhibits sorted by cost from least to greatest.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits ORDER BY cost ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY cost ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "060573c57829f3ce6e8f_en", "language": "en", "backend": "supabase", "operation": "not_null", "question": "Present the data from exhibits where item_name is not null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE item_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE item_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "060573c57829f3ce6e8f_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "not_null", "question": "Find the rows in exhibits where item_name is not null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE item_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE item_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "92e36c8a6ca6e7aa2298_hinglish", "language": "hinglish", "backend": "supabase", "operation": "group_count", "question": "reservations me pratyek category ke liye rows ki sankhya dhundo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e3d12f1bf51bbf54a331_en", "language": "en", "backend": "supabase", "operation": "contains", "question": "List all rows in specimens where description is equal to North in a case-insensitive way.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE LOWER(description) LIKE LOWER('%North%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE LOWER(description) LIKE LOWER('%North%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "fd2a795a1557dc1487c6_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "or", "question": "Display rows from reservations where city is Electronics or Pune.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE city IN ('Electronics', 'Pune');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE city IN ('Electronics', 'Pune');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "fd2a795a1557dc1487c6_hinglish", "language": "hinglish", "backend": "supabase", "operation": "or", "question": "reservations mein city Electronics ya Pune wali rows dikhao na.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE city IN ('Electronics', 'Pune');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE city IN ('Electronics', 'Pune');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "3ad8170567694db8c8f9_en", "language": "en", "backend": "supabase", "operation": "not_null", "question": "Present the data from inspections where display_name is not null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE display_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE display_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "3ad8170567694db8c8f9_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "not_null", "question": "Find the rows in inspections where display_name is not null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE display_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE display_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5b5b87a5a58feab9d244_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "year", "question": "Fetch reservations entries where recorded_at year is 2021.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE EXTRACT(YEAR FROM recorded_at) = 2021;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE EXTRACT(YEAR FROM recorded_at) = 2021;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5b5b87a5a58feab9d244_hinglish", "language": "hinglish", "backend": "supabase", "operation": "year", "question": "reservations mein recorded_at mein 2021 wale records dikhao.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE EXTRACT(YEAR FROM recorded_at) = 2021;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE EXTRACT(YEAR FROM recorded_at) = 2021;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "37a16740c32af37d3b15_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "lte", "question": "show the reservations table rows where score is at most 50", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE score <= 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE score <= 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0c15b441e3e3ee8bb0df_en", "language": "en", "backend": "supabase", "operation": "not_null", "question": "Present the data from reservations where name is not null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0c15b441e3e3ee8bb0df_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "not_null", "question": "Find the rows in reservations where name is not null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0c15b441e3e3ee8bb0df_hinglish", "language": "hinglish", "backend": "supabase", "operation": "not_null", "question": "reservations ka data present karo jahan name null nahi hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4dac1dcf2ef14bfee2d2_hinglish", "language": "hinglish", "backend": "supabase", "operation": "group_count", "question": "reservations me pratyek category ke liye rows ki sankhya dhundo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "56189c5cdc014dd771be_en", "language": "en", "backend": "supabase", "operation": "lt", "question": "Query reservations for rows where balance is less than 50.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE balance < 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE balance < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ec5a4c33f3a4c1fdfef3_en", "language": "en", "backend": "supabase", "operation": "group_sum", "question": "Please provide the total rating for each city in reservations.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT city, SUM(rating) FROM reservations GROUP BY city;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT city, SUM(rating) FROM reservations GROUP BY city;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "26db633822b8319f83ec_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "contains", "question": "List reservations rows where name equals active, ignore case differences.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE LOWER(name) LIKE LOWER('%active%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE LOWER(name) LIKE LOWER('%active%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "060573c57829f3ce6e8f_hinglish", "language": "hinglish", "backend": "supabase", "operation": "not_null", "question": "exhibits ka data present karo jahan item_name null nahi hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE item_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE item_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7dd163238936d96d0bd4_en", "language": "en", "backend": "supabase", "operation": "distinct", "question": "I want to see the distinct city values available in reservations.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT DISTINCT city FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT city FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7dd163238936d96d0bd4_hinglish", "language": "hinglish", "backend": "supabase", "operation": "distinct", "question": "Mujhe reservations mein available city ke distinct values dekhne hain.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT DISTINCT city FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT city FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "3ad8170567694db8c8f9_hinglish", "language": "hinglish", "backend": "supabase", "operation": "not_null", "question": "inspections ka data present karo jahan display_name null nahi hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE display_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE display_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5b5b87a5a58feab9d244_en", "language": "en", "backend": "supabase", "operation": "year", "question": "Show me the reservations rows for 2021 using recorded_at.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE EXTRACT(YEAR FROM recorded_at) = 2021;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE EXTRACT(YEAR FROM recorded_at) = 2021;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "37a16740c32af37d3b15_en", "language": "en", "backend": "supabase", "operation": "lte", "question": "Query reservations for rows where score is not greater than 50.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE score <= 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE score <= 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "32876ff450ca027d658e_hi", "language": "hi", "backend": "supabase", "operation": "count", "question": "specimens में कितनी लाइनें मौजूद हैं?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "175564da35a6c139e5d8_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "contains", "question": "List inspections rows where display_name equals North, ignore case differences.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE LOWER(display_name) LIKE LOWER('%North%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE LOWER(display_name) LIKE LOWER('%North%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ec5a4c33f3a4c1fdfef3_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "group_sum", "question": "I need to see the total rating for each city in reservations", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT city, SUM(rating) FROM reservations GROUP BY city;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT city, SUM(rating) FROM reservations GROUP BY city;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ec5a4c33f3a4c1fdfef3_hinglish", "language": "hinglish", "backend": "supabase", "operation": "group_sum", "question": "reservations me city ke hisaab se rating ka total do please.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT city, SUM(rating) FROM reservations GROUP BY city;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT city, SUM(rating) FROM reservations GROUP BY city;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c4f25575abfd24e3cd63_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "lte", "question": "show the reservations table rows where cost is at most 1", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE cost <= 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE cost <= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b30330bb2c17b32c6883_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sort_asc", "question": "can i see all rows from exhibits sorted by cost from small to large", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits ORDER BY cost ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY cost ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b30330bb2c17b32c6883_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sort_asc", "question": "exhibits se saara data cost ke badhte hue order mein lo aur dikhao.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits ORDER BY cost ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY cost ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "645fd36c186709e64922_en", "language": "en", "backend": "supabase", "operation": "between", "question": "Select all items from specimens where the amount column has a value between 1000 and 1010, inclusive.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE amount BETWEEN 1000 AND 1010;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE amount BETWEEN 1000 AND 1010;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "fd2a795a1557dc1487c6_en", "language": "en", "backend": "supabase", "operation": "or", "question": "Bring up rows from reservations where city is either Electronics or Pune.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE city IN ('Electronics', 'Pune');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE city IN ('Electronics', 'Pune');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4745f53221e07d4f5ae1_hinglish", "language": "hinglish", "backend": "supabase", "operation": "contains", "question": "Output karo specimens mein display_name mein Pune match hone wali rows, case insensitive raho.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE LOWER(display_name) LIKE LOWER('%Pune%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE LOWER(display_name) LIKE LOWER('%Pune%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "37a16740c32af37d3b15_hinglish", "language": "hinglish", "backend": "supabase", "operation": "lte", "question": "reservations se wo rows dhundo jaha score 50 se bada nahi hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE score <= 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE score <= 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7d18561ec942f18f1299_en", "language": "en", "backend": "supabase", "operation": "count_eq", "question": "Provide the row count for reservations where status is O'Reilly.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM reservations WHERE status = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM reservations WHERE status = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7d18561ec942f18f1299_hinglish", "language": "hinglish", "backend": "supabase", "operation": "count_eq", "question": "Please count the rows in reservations where status is O'Reilly.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM reservations WHERE status = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM reservations WHERE status = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0ce22f4c8bc7554d586d_en", "language": "en", "backend": "supabase", "operation": "lt", "question": "Query reservations for rows where score is less than 50.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE score < 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE score < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f35f7a78a14e903920ee_en", "language": "en", "backend": "supabase", "operation": "month", "question": "Select entries from reservations where order_date corresponds to 3.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE EXTRACT(MONTH FROM order_date) = 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE EXTRACT(MONTH FROM order_date) = 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c4f25575abfd24e3cd63_en", "language": "en", "backend": "supabase", "operation": "lte", "question": "Query reservations for rows where cost is not greater than 1.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE cost <= 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE cost <= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "29bdf95ec36d6924a576_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "contains", "question": "List exhibits rows where item_name equals O'Reilly, ignore case differences.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE LOWER(item_name) LIKE LOWER('%O''Reilly%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE LOWER(item_name) LIKE LOWER('%O''Reilly%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "56189c5cdc014dd771be_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "lt", "question": "can you select the rows from reservations where balance is less than 50?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE balance < 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE balance < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "56189c5cdc014dd771be_hinglish", "language": "hinglish", "backend": "supabase", "operation": "lt", "question": "reservations se un rows ko query karo jahan balance 50 se kam hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE balance < 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE balance < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0b8689cf8e381cbcfe86_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "and", "question": "find rows in reservations where department is pending and price is above 10000", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE department = 'pending' AND price > 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE department = 'pending' AND price > 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2e2ac3c5dcf95429daaa_hi", "language": "hi", "backend": "supabase", "operation": "null", "question": "specimens से title null वाली पंक्तियाँ चुनें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1a1af4deb475f0224a59_hinglish", "language": "hinglish", "backend": "supabase", "operation": "bottom", "question": "price ke badhte order me inspections ki 10 sabse kam rows, lowest se shuru karke, lao.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections ORDER BY price ASC LIMIT 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY price ASC LIMIT 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f35f7a78a14e903920ee_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "month", "question": "extract rows from reservations where order_date month is 3", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE EXTRACT(MONTH FROM order_date) = 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE EXTRACT(MONTH FROM order_date) = 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c4f25575abfd24e3cd63_hinglish", "language": "hinglish", "backend": "supabase", "operation": "lte", "question": "reservations se wo rows dhundo jaha cost 1 se bada nahi hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE cost <= 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE cost <= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "84b620e6d482597eea06_hi", "language": "hi", "backend": "supabase", "operation": "project", "question": "सिर्फ exhibits का display_name दिखाइए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT display_name FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT display_name FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "645fd36c186709e64922_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "between", "question": "pull the rows from specimens where amount is greater than or equal to 1000 and less than or equal to 1010", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE amount BETWEEN 1000 AND 1010;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE amount BETWEEN 1000 AND 1010;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4745f53221e07d4f5ae1_en", "language": "en", "backend": "supabase", "operation": "contains", "question": "List all rows in specimens where display_name is equal to Pune in a case-insensitive way.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE LOWER(display_name) LIKE LOWER('%Pune%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE LOWER(display_name) LIKE LOWER('%Pune%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "358364da33322bbf17e2_hi", "language": "hi", "backend": "supabase", "operation": "project_eq", "question": "exhibits से title की सूची दें जहाँ city Pune है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT title FROM exhibits WHERE city = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT title FROM exhibits WHERE city = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "dd6cdefd15edcdcddb76_hinglish", "language": "hinglish", "backend": "supabase", "operation": "bottom", "question": "salary ke badhte order me exhibits ki 5 sabse kam rows, lowest se shuru karke, lao.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits ORDER BY salary ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY salary ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5689f342b9bf3175e2fe_hi", "language": "hi", "backend": "supabase", "operation": "all", "question": "specimens के सभी रिकॉर्ड प्राप्त करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0abe13c841fd7c3ac885_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "having", "question": "Tell me the region groups in inspections with more than 5000 rows and their count values.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT region, COUNT(*) FROM inspections GROUP BY region HAVING COUNT(*) > 5000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT region, COUNT(*) FROM inspections GROUP BY region HAVING COUNT(*) > 5000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0abe13c841fd7c3ac885_hinglish", "language": "hinglish", "backend": "supabase", "operation": "having", "question": "Please inspections me 5000 se zyada rows wale region groups aur unki count ko display karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT region, COUNT(*) FROM inspections GROUP BY region HAVING COUNT(*) > 5000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT region, COUNT(*) FROM inspections GROUP BY region HAVING COUNT(*) > 5000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0b8689cf8e381cbcfe86_en", "language": "en", "backend": "supabase", "operation": "and", "question": "Show me the reservations rows where department is pending and price exceeds 10000.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE department = 'pending' AND price > 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE department = 'pending' AND price > 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "916ff7ce3eea9aa64393_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "missing_value", "question": "execute filter on reservations for kind, but i have not specified a value", "expected": {"action": "clarify", "question": "Which value should kind equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should kind equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "2984a6707d4e07f81695_hi", "language": "hi", "backend": "supabase", "operation": "write", "question": "exhibits का पूरा कंटेंट डिलीट करें।", "expected": {"action": "clarify", "question": "यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "0ce22f4c8bc7554d586d_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "lt", "question": "can you select the rows from reservations where score is less than 50?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE score < 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE score < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0ce22f4c8bc7554d586d_hinglish", "language": "hinglish", "backend": "supabase", "operation": "lt", "question": "reservations se un rows ko query karo jahan score 50 se kam hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE score < 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE score < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f35f7a78a14e903920ee_hinglish", "language": "hinglish", "backend": "supabase", "operation": "month", "question": "Show me the data from reservations where order_date ka month 3 hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE EXTRACT(MONTH FROM order_date) = 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE EXTRACT(MONTH FROM order_date) = 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "26db633822b8319f83ec_hinglish", "language": "hinglish", "backend": "supabase", "operation": "contains", "question": "Output karo reservations mein name mein active match hone wali rows, case insensitive raho.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE LOWER(name) LIKE LOWER('%active%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE LOWER(name) LIKE LOWER('%active%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2ad3e251c936135e2005_hi", "language": "hi", "backend": "supabase", "operation": "all", "question": "specimens के सभी रिकॉर्ड प्राप्त करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "645fd36c186709e64922_hinglish", "language": "hinglish", "backend": "supabase", "operation": "between", "question": "specimens se wo saare items select karo jahan amount column ka value 1000 aur 1010 ke beech hai, inclusive.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE amount BETWEEN 1000 AND 1010;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE amount BETWEEN 1000 AND 1010;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e4a81e1122226c7b26d9_hi", "language": "hi", "backend": "supabase", "operation": "describe", "question": "टेबल specimens के कॉलम संरचना और टाइप्स दिखाइए।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "e9cfe7f6c91a2434d315_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "final", "question": "db tool returned 102 rows. tell me the total.", "expected": {"action": "answer", "text": "The tool returned 102 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 102 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "e9cfe7f6c91a2434d315_hinglish", "language": "hinglish", "backend": "supabase", "operation": "final", "question": "Kitni rows aayin database tool se? Count hai 102.", "expected": {"action": "answer", "text": "Tool ne 102 rows return ki."}, "output": "{\"action\":\"answer\",\"text\":\"Tool ne 102 rows return ki.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "175564da35a6c139e5d8_hinglish", "language": "hinglish", "backend": "supabase", "operation": "contains", "question": "Output karo inspections mein display_name mein North match hone wali rows, case insensitive raho.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE LOWER(display_name) LIKE LOWER('%North%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE LOWER(display_name) LIKE LOWER('%North%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0b8689cf8e381cbcfe86_hinglish", "language": "hinglish", "backend": "supabase", "operation": "and", "question": "reservations mein wo rows dhundo jahan department pending hai aur price 10000 se upar hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE department = 'pending' AND price > 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE department = 'pending' AND price > 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d735c1ac1979ff1e8620_hi", "language": "hi", "backend": "supabase", "operation": "sum", "question": "inspections में price के सभी मानों का योग करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT SUM(price) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT SUM(price) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "eb982cf071094adaca4f_hi", "language": "hi", "backend": "supabase", "operation": "project_eq", "question": "reservations से name की सूची दें जहाँ category Delhi है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT name FROM reservations WHERE category = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT name FROM reservations WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "85e339da2fb28bae5395_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "date_after", "question": "Query rows in reservations where order_date is 2024-10-15 onwards.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE order_date >= '2024-10-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE order_date >= '2024-10-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "65666802d11552ba1fc2_hi", "language": "hi", "backend": "supabase", "operation": "write", "question": "inspections का पूरा कंटेंट डिलीट करें।", "expected": {"action": "clarify", "question": "यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "e91a7141af66518c19a2_hi", "language": "hi", "backend": "supabase", "operation": "max", "question": "inspections में cost का सबसे बड़ा नंबर क्या है?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MAX(cost) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(cost) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0ffff93d2f774a97e3f3_hi", "language": "hi", "backend": "supabase", "operation": "min", "question": "specimens में price का न्यूनतम परिमाण क्या है?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MIN(price) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(price) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "26db633822b8319f83ec_en", "language": "en", "backend": "supabase", "operation": "contains", "question": "List all rows in reservations where name is equal to active in a case-insensitive way.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE LOWER(name) LIKE LOWER('%active%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE LOWER(name) LIKE LOWER('%active%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "30dd94e63a065d87109e_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sql_error", "question": "The last one failed because of a unknown column. Please inspect specimens schema before retrying.", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "specimens"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"specmens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "578a0a8a98b724c87050_hi", "language": "hi", "backend": "supabase", "operation": "list_tables", "question": "इस डेटाबेस की तालिकाओं का विवरण दीजिए।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "916ff7ce3eea9aa64393_en", "language": "en", "backend": "supabase", "operation": "missing_value", "question": "Execute a filter on reservations based on kind, but I have not specified a value.", "expected": {"action": "clarify", "question": "Which value should kind equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should kind equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "1395b37e5ba28fa3c134_hi", "language": "hi", "backend": "supabase", "operation": "min", "question": "inspections में amount का न्यूनतम परिमाण क्या है?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MIN(amount) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(amount) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "29bdf95ec36d6924a576_hinglish", "language": "hinglish", "backend": "supabase", "operation": "contains", "question": "Output karo exhibits mein item_name mein O'Reilly match hone wali rows, case insensitive raho.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE LOWER(item_name) LIKE LOWER('%O''Reilly%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE LOWER(item_name) LIKE LOWER('%O''Reilly%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6f162db917ded36d0df0_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sql_error", "question": "The last one failed because of a unknown column. Please inspect specimens schema before retrying.", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "specimens"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"specmens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "175564da35a6c139e5d8_en", "language": "en", "backend": "supabase", "operation": "contains", "question": "List all rows in inspections where display_name is equal to North in a case-insensitive way.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE LOWER(display_name) LIKE LOWER('%North%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE LOWER(display_name) LIKE LOWER('%North%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ac83eae59947f137269f_hi", "language": "hi", "backend": "supabase", "operation": "write", "question": "reservations का पूरा कंटेंट डिलीट करें।", "expected": {"action": "clarify", "question": "यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "e9cfe7f6c91a2434d315_en", "language": "en", "backend": "supabase", "operation": "final", "question": "The database tool returned 102 rows. Can you tell me the number?", "expected": {"action": "answer", "text": "The tool returned 102 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 102 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "7a843b154fbd48aac50b_hi", "language": "hi", "backend": "supabase", "operation": "all", "question": "inspections के सभी रिकॉर्ड प्राप्त करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4f5c410b4b3379c58420_hi", "language": "hi", "backend": "supabase", "operation": "or", "question": "specimens में category pending या North वाली पंक्तियाँ दिखाइए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE category IN ('pending', 'North');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE category IN ('pending', 'North');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0abe13c841fd7c3ac885_en", "language": "en", "backend": "supabase", "operation": "having", "question": "Give me the region groups in inspections where the row count is higher than 5000, along with the count values.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT region, COUNT(*) FROM inspections GROUP BY region HAVING COUNT(*) > 5000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT region, COUNT(*) FROM inspections GROUP BY region HAVING COUNT(*) > 5000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "940596942f7c5e4bf758_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "list_tables", "question": "tell me the tables in this database", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "0c15b441e3e3ee8bb0df_hi", "language": "hi", "backend": "supabase", "operation": "not_null", "question": "reservations का डेटा दिखाएं जहाँ name null नहीं है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "29bdf95ec36d6924a576_en", "language": "en", "backend": "supabase", "operation": "contains", "question": "List all rows in exhibits where item_name is equal to O'Reilly in a case-insensitive way.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE LOWER(item_name) LIKE LOWER('%O''Reilly%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE LOWER(item_name) LIKE LOWER('%O''Reilly%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "30dd94e63a065d87109e_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sql_error", "question": "Pichla query fail raha kyunki ek column unknown tha. Dobara try karne se pehle specimens schema verify karo.", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "specimens"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"sicmens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "060573c57829f3ce6e8f_hi", "language": "hi", "backend": "supabase", "operation": "not_null", "question": "exhibits का डेटा दिखाएं जहाँ item_name null नहीं है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE item_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE item_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "85e339da2fb28bae5395_en", "language": "en", "backend": "supabase", "operation": "date_after", "question": "Extract rows from reservations where order_date falls on 2024-10-15 or beyond.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE order_date >= '2024-10-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE order_date >= '2024-10-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "3ad8170567694db8c8f9_hi", "language": "hi", "backend": "supabase", "operation": "not_null", "question": "inspections का डेटा दिखाएं जहाँ display_name null नहीं है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE display_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE display_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "940596942f7c5e4bf758_en", "language": "en", "backend": "supabase", "operation": "list_tables", "question": "Display the schema tables in this database.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "940596942f7c5e4bf758_hinglish", "language": "hinglish", "backend": "supabase", "operation": "list_tables", "question": "Is database ki tables ka detail do.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "6f162db917ded36d0df0_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sql_error", "question": "Pichla query fail raha kyunki ek column unknown tha. Dobara try karne se pehle specimens schema verify karo.", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "specimens"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"saarmens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "5a8f2e207d0efe5b018c_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "list_tables", "question": "tell me the tables in this database", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "30dd94e63a065d87109e_en", "language": "en", "backend": "mysql", "operation": "sql_error", "question": "An unknown column caused the last query to fail; check specimens schema before your next attempt.", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "specimens"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"specmens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "916ff7ce3eea9aa64393_hinglish", "language": "hinglish", "backend": "supabase", "operation": "missing_value", "question": "kind ke basis pe reservations pe ek filter execute karo, lekin maine koi value specify nahi ki hai.", "expected": {"action": "clarify", "question": "kind ki kaunsi value chahiye?"}, "output": "{\"action\":\"clarify\",\"question\":\"kind ki kaunsi value chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "bcd7a532a06274ddddee_hi", "language": "hi", "backend": "supabase", "operation": "or", "question": "exhibits में status pending या South वाली पंक्तियाँ दिखाइए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE status IN ('pending', 'South');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE status IN ('pending', 'South');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8a456280df4a711c3cda_hi", "language": "hi", "backend": "supabase", "operation": "year", "question": "inspections में created_at में 2024 वाले रिकॉर्ड दिखाएं।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE EXTRACT(YEAR FROM created_at) = 2024;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE EXTRACT(YEAR FROM created_at) = 2024;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "aa4f0e1f29573e237ee1_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "list_tables", "question": "tell me the tables in this database", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "a7afc1ab21c25c700603_hi", "language": "hi", "backend": "supabase", "operation": "or", "question": "exhibits में status Mumbai या Electronics वाली पंक्तियाँ दिखाइए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE status IN ('Mumbai', 'Electronics');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE status IN ('Mumbai', 'Electronics');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6f162db917ded36d0df0_en", "language": "en", "backend": "mysql", "operation": "sql_error", "question": "An unknown column caused the last query to fail; check specimens schema before your next attempt.", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "specimens"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"specmens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "ec5a4c33f3a4c1fdfef3_hi", "language": "hi", "backend": "supabase", "operation": "group_sum", "question": "reservations में city के अनुसार rating का कुल कृपया दें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT city, SUM(rating) FROM reservations GROUP BY city;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT city, SUM(rating) FROM reservations GROUP BY city;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5a8f2e207d0efe5b018c_en", "language": "en", "backend": "supabase", "operation": "list_tables", "question": "Display the schema tables in this database.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "5a8f2e207d0efe5b018c_hinglish", "language": "hinglish", "backend": "supabase", "operation": "list_tables", "question": "Is database ki tables ka detail do.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "aa4f0e1f29573e237ee1_en", "language": "en", "backend": "supabase", "operation": "list_tables", "question": "Display the schema tables in this database.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "aa4f0e1f29573e237ee1_hinglish", "language": "hinglish", "backend": "supabase", "operation": "list_tables", "question": "Is database ki tables ka detail do.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "c15c59356c7863f0b7e9_hi", "language": "hi", "backend": "supabase", "operation": "eq", "question": "specimens से वे सभी पंक्तियाँ लाएँ जहाँ city का मान Books है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE city = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE city = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8ebc9e868ee558a8ddfe_en", "language": "en", "backend": "supabase", "operation": "ambiguous", "question": "Print the best available records in specimens.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "8ebc9e868ee558a8ddfe_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "ambiguous", "question": "Give me the top records in specimens.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "0b8db53d711baed4f4ed_hi", "language": "hi", "backend": "supabase", "operation": "gte", "question": "specimens से वे पंक्तियाँ दिखाएँ जहाँ cost 10 या उससे बड़ा है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE cost >= 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE cost >= 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b564e1a7b1e7fc6d2ab4_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "avg", "question": "determine the mean score in specimens", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT AVG(score) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT AVG(score) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "85e339da2fb28bae5395_hinglish", "language": "hinglish", "backend": "supabase", "operation": "date_after", "question": "reservations mein se wo rows chuno jinke order_date mein 2024-10-15 ya uske baad ki date hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE order_date >= '2024-10-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE order_date >= '2024-10-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "28530a7607feca135b73_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sum", "question": "i want the total price of specimens", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT SUM(price) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmens\",\"query\":\"SELECT SUM(price) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "fd2a795a1557dc1487c6_hi", "language": "hi", "backend": "supabase", "operation": "or", "question": "reservations में city Electronics या Pune वाली पंक्तियाँ दिखाइए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE city IN ('Electronics', 'Pune');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE city IN ('Electronics', 'Pune');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5b5b87a5a58feab9d244_hi", "language": "hi", "backend": "supabase", "operation": "year", "question": "reservations में recorded_at में 2021 वाले रिकॉर्ड दिखाएं।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE EXTRACT(YEAR FROM recorded_at) = 2021;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE EXTRACT(YEAR FROM recorded_at) = 2021;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8ebc9e868ee558a8ddfe_hinglish", "language": "hinglish", "backend": "supabase", "operation": "ambiguous", "question": "specimens mein available best records print karo.", "expected": {"action": "clarify", "question": "Best ka matlab kya hai? Kaunsa column aur order?"}, "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "7e1cd79fe94cad9c7a9e_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "count_eq", "question": "how many specimens rows where region is Books", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT COUNT(*) FROM specimens WHERE region = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT COUNT(*) FROM specimens WHERE region = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "ac6cc6f05e6d013971e5_en", "language": "en", "backend": "supabase", "operation": "max", "question": "What is the largest number for quantity in specimens?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT MAX(quantity) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT MAX(quantity) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "ac6cc6f05e6d013971e5_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "max", "question": "what is the upper bound of quantity in specimens?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT MAX(quantity) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmens\",\"query\":\"SELECT MAX(quantity) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "35406fa024e66bcb5dd7_en", "language": "en", "backend": "supabase", "operation": "lt", "question": "Query specimens for rows where score is less than 10000.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE score < 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmens\",\"query\":\"SELECT * FROM specimens WHERE score < 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "c8fd1555fd64b08f1241_hi", "language": "hi", "backend": "supabase", "operation": "eq", "question": "inspections से वे सभी पंक्तियाँ लाएँ जहाँ city का मान Electronics है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE city = 'Electronics';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE city = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b564e1a7b1e7fc6d2ab4_en", "language": "en", "backend": "supabase", "operation": "avg", "question": "Return the average score in specimens.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT AVG(score) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT AVG(score) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "51f14088477a5f6be9c5_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "describe", "question": "Look up the columns and types for specimens.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "96a74866d51ea7899cd8_hi", "language": "hi", "backend": "supabase", "operation": "gte", "question": "specimens से वे पंक्तियाँ दिखाएँ जहाँ score 20 या उससे बड़ा है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE score >= 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE score >= 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1110104f12e10c5a007a_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sum", "question": "i want the total amount of inspections", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT SUM(amount) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT SUM(amount) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "797213190eddac109b96_hi", "language": "hi", "backend": "supabase", "operation": "sql_error", "question": "पिछला क्वेरी विफल रहा क्योंकि एक कॉलम अज्ञात था। दोबारा प्रयास करने से पहले inspections स्कीमा की पुष्टि करें।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "28530a7607feca135b73_en", "language": "en", "backend": "supabase", "operation": "sum", "question": "Output the total price for the specimens entity.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT SUM(price) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT SUM(price) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "6b9bacf4d9aa380af183_hi", "language": "hi", "backend": "supabase", "operation": "ambiguous", "question": "reservations में उपलब्ध सर्वोत्तम रिकॉर्ड प्रिंट करें।", "expected": {"action": "clarify", "question": "सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?"}, "output": "{\"action\":\"clarify\",\"question\":\"सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "924dbd9ad819c0a58b24_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sum", "question": "i want the total salary of inspections", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT SUM(salary) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT SUM(salary) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f7257c4a2eded61f5c86_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sort_desc", "question": "retrieve all records from specimens sorted by score high to low", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens ORDER BY score DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmens\",\"query\":\"SELECT * FROM specimens ORDER BY score DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "4b73bea165fb5f9555f3_hi", "language": "hi", "backend": "supabase", "operation": "sql_error", "question": "पिछला क्वेरी विफल रहा क्योंकि एक कॉलम अज्ञात था। दोबारा प्रयास करने से पहले inspections स्कीमा की पुष्टि करें।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "82c104614df1ce6e2762_en", "language": "en", "backend": "supabase", "operation": "gt", "question": "Bring up the rows in specimens where salary is over 100.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE salary > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmens\",\"query\":\"SELECT * FROM specimens WHERE salary > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "40464b00088c9159e3d0_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "count", "question": "how many lines in inspections?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT COUNT(*) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT COUNT(*) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b564e1a7b1e7fc6d2ab4_hinglish", "language": "hinglish", "backend": "supabase", "operation": "avg", "question": "specimens me score ka average return karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT AVG(score) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT AVG(score) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0a2ec84ba0e594d94b9e_hi", "language": "hi", "backend": "supabase", "operation": "sql_error", "question": "पिछला क्वेरी विफल रहा क्योंकि एक कॉलम अज्ञात था। दोबारा प्रयास करने से पहले reservations स्कीमा की पुष्टि करें।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "bf7bc588f62f075ec1ca_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "count", "question": "how many lines in exhibits?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT COUNT(*) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT COUNT(*) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5c6734bdfead38b51752_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "gte", "question": "output records from specimens where quantity is not below 10000", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE quantity >= 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens WHERE quantity >= 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "ac6cc6f05e6d013971e5_hinglish", "language": "hinglish", "backend": "supabase", "operation": "max", "question": "specimens mein quantity ka sabse bada number kya hai?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT MAX(quantity) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT MAX(quantity) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "82c104614df1ce6e2762_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "gt", "question": "show me the entries in specimens where salary is strictly greater than 100", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE salary > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens WHERE salary > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "40464b00088c9159e3d0_hinglish", "language": "hinglish", "backend": "supabase", "operation": "count", "question": "Batao inspections mein kitni entries hain.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT COUNT(*) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT COUNT(*) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "80ed46f6e275ed01c6c1_hi", "language": "hi", "backend": "supabase", "operation": "count_eq", "question": "exhibits में category Pune वाले रिकॉर्ड्स की संख्या दीजिए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM exhibits WHERE category = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM exhibits WHERE category = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1896aafc31bd0a7d24ec_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sum", "question": "i want the total balance of exhibits", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT SUM(balance) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT SUM(balance) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ed474bd8a459cbfaa5d2_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "year", "question": "Fetch specimens entries where recorded_at year is 2021.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE EXTRACT(YEAR FROM recorded_at) = 2021;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmens\",\"query\":\"SELECT * FROM specimens WHERE EXTRACT(YEAR FROM recorded_at) = 2021;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "51f14088477a5f6be9c5_en", "language": "en", "backend": "supabase", "operation": "describe", "question": "Reveal the column structure and types for specimens.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "fea44ee93f1cea0883dd_hi", "language": "hi", "backend": "supabase", "operation": "count_eq", "question": "exhibits में category Delhi वाले रिकॉर्ड्स की संख्या दीजिए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM exhibits WHERE category = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM exhibits WHERE category = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7dd163238936d96d0bd4_hi", "language": "hi", "backend": "supabase", "operation": "distinct", "question": "मुझे reservations में उपलब्ध city के विशिष्ट मान देखने हैं।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT DISTINCT city FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT city FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4fb8670235a435c82c88_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "describe", "question": "Look up the columns and types for exhibits.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_exhibits", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "864b7a7bf739071bb030_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "and", "question": "find rows in specimens where status is Pune and balance is above 0", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE status = 'Pune' AND balance > 0;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens WHERE status = 'Pune' AND balance > 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "1110104f12e10c5a007a_en", "language": "en", "backend": "supabase", "operation": "sum", "question": "Output the total amount for the inspections entity.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT SUM(amount) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT SUM(amount) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "37a16740c32af37d3b15_hi", "language": "hi", "backend": "supabase", "operation": "lte", "question": "reservations से वे पंक्तियां खोजें जहां score 50 से बड़ा नहीं है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE score <= 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE score <= 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bf7bc588f62f075ec1ca_hinglish", "language": "hinglish", "backend": "supabase", "operation": "count", "question": "Batao exhibits mein kitni entries hain.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT COUNT(*) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT COUNT(*) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "41f5fbcefb9d550068a0_hi", "language": "hi", "backend": "supabase", "operation": "missing_schema", "question": "कृपया specimens के लिए SQL लिखने से पहले उपलब्ध तालिकाओं का पता लगाएं, क्योंकि मैंने डेटाबेस स्कीमा नहीं दी है।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "924dbd9ad819c0a58b24_en", "language": "en", "backend": "supabase", "operation": "sum", "question": "Output the total salary for the inspections entity.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT SUM(salary) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT SUM(salary) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f7257c4a2eded61f5c86_en", "language": "en", "backend": "supabase", "operation": "sort_desc", "question": "Display the entire table specimens sorted by score from highest to lowest.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens ORDER BY score DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens ORDER BY score DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "7e1cd79fe94cad9c7a9e_en", "language": "en", "backend": "supabase", "operation": "count_eq", "question": "Provide the row count for specimens where region is Books.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT COUNT(*) FROM specimens WHERE region = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT COUNT(*) FROM specimens WHERE region = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "7e1cd79fe94cad9c7a9e_hinglish", "language": "hinglish", "backend": "supabase", "operation": "count_eq", "question": "Please count the rows in specimens where region is Books.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT COUNT(*) FROM specimens WHERE region = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT COUNT(*) FROM specimens WHERE region = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "35406fa024e66bcb5dd7_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "lt", "question": "can you select the rows from specimens where score is less than 10000?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE score < 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmens\",\"query\":\"SELECT * FROM specimens WHERE score < 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "ed474bd8a459cbfaa5d2_en", "language": "en", "backend": "supabase", "operation": "year", "question": "Show me the specimens rows for 2021 using recorded_at.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE EXTRACT(YEAR FROM recorded_at) = 2021;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens WHERE EXTRACT(YEAR FROM recorded_at) = 2021;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "223765dc6e033f0ab2fd_en", "language": "en", "backend": "supabase", "operation": "month", "question": "Select entries from specimens where created_at corresponds to 9.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE EXTRACT(MONTH FROM created_at) = 9;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens WHERE EXTRACT(MONTH FROM created_at) = 9;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "51f14088477a5f6be9c5_hinglish", "language": "hinglish", "backend": "supabase", "operation": "describe", "question": "Mujhe specimens ke columns aur types ke baare mein info do.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "92e36c8a6ca6e7aa2298_hi", "language": "hi", "backend": "supabase", "operation": "group_count", "question": "reservations में हर category के लिए पंक्तियों की संख्या खोजें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "864b7a7bf739071bb030_en", "language": "en", "backend": "supabase", "operation": "and", "question": "Show me the specimens rows where status is Pune and balance exceeds 0.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE status = 'Pune' AND balance > 0;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens WHERE status = 'Pune' AND balance > 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "2ffce857234ca4c32ec9_hinglish", "language": "hinglish", "backend": "supabase", "operation": "project", "question": "bas reservations ka title do please", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT title FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT title FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8fa63f2c53b1bc742d49_hinglish", "language": "hinglish", "backend": "supabase", "operation": "null", "question": "inspections se title null wali rows select karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "28530a7607feca135b73_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sum", "question": "specimens entity ke liye price ka total output do.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT SUM(price) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT SUM(price) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4dac1dcf2ef14bfee2d2_hi", "language": "hi", "backend": "supabase", "operation": "group_count", "question": "reservations में हर category के लिए पंक्तियों की संख्या खोजें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5c6734bdfead38b51752_en", "language": "en", "backend": "supabase", "operation": "gte", "question": "Show the data from specimens where quantity is 10000 or above.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE quantity >= 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens WHERE quantity >= 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "f7257c4a2eded61f5c86_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sort_desc", "question": "specimens ki poori table score highest se lowest order mein dikhao.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens ORDER BY score DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens ORDER BY score DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "44135783de9bd091bfe5_hi", "language": "hi", "backend": "supabase", "operation": "missing_schema", "question": "कृपया inspections के लिए SQL लिखने से पहले उपलब्ध तालिकाओं का पता लगाएं, क्योंकि मैंने डेटाबेस स्कीमा नहीं दी है।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_inspections", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_inspections\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "28a7ce61b1233483bf73_hi", "language": "hi", "backend": "supabase", "operation": "sort_desc", "question": "specimens की पूरी सारणी salary के उच्चतम से निम्नतम क्रम में दिखाएँ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens ORDER BY salary DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c4f25575abfd24e3cd63_hi", "language": "hi", "backend": "supabase", "operation": "lte", "question": "reservations से वे पंक्तियां खोजें जहां cost 1 से बड़ा नहीं है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE cost <= 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE cost <= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8b1939fffee21884f667_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "null", "question": "Show me the rows from specimens where item_name is null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE item_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "8b1939fffee21884f667_hinglish", "language": "hinglish", "backend": "supabase", "operation": "null", "question": "specimens se item_name null wali rows select karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE item_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1896aafc31bd0a7d24ec_en", "language": "en", "backend": "supabase", "operation": "sum", "question": "Output the total balance for the exhibits entity.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT SUM(balance) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT SUM(balance) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ed474bd8a459cbfaa5d2_hinglish", "language": "hinglish", "backend": "supabase", "operation": "year", "question": "specimens mein recorded_at mein 2021 wale records dikhao.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE EXTRACT(YEAR FROM recorded_at) = 2021;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE EXTRACT(YEAR FROM recorded_at) = 2021;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "223765dc6e033f0ab2fd_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "month", "question": "extract rows from specimens where created_at month is 9", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE EXTRACT(MONTH FROM created_at) = 9;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens WHERE EXTRACT(MONTH FROM created_at) = 9;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "6db4ac493194a52cbd3a_hi", "language": "hi", "backend": "supabase", "operation": "missing_schema", "question": "कृपया reservations के लिए SQL लिखने से पहले उपलब्ध तालिकाओं का पता लगाएं, क्योंकि मैंने डेटाबेस स्कीमा नहीं दी है।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_reservations", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_reservations\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "fd0bab1cc08d7298f131_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "top", "question": "I need 20 rows from exhibits with salary sorted descending.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4fb8670235a435c82c88_en", "language": "en", "backend": "supabase", "operation": "describe", "question": "Reveal the column structure and types for exhibits.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_exhibits", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "1110104f12e10c5a007a_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sum", "question": "inspections entity ke liye amount ka total output do.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT SUM(amount) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT SUM(amount) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2ffce857234ca4c32ec9_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "project", "question": "display title from reservations and nothing else", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT title FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT title FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8fa63f2c53b1bc742d49_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "null", "question": "Show me the rows from inspections where title is null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "56189c5cdc014dd771be_hi", "language": "hi", "backend": "supabase", "operation": "lt", "question": "reservations से उन पंक्तियों का क्वेरी करें जहाँ balance 50 से कम है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE balance < 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE balance < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "3eea55cd4532636ddffe_en", "language": "en", "backend": "supabase", "operation": "not_null", "question": "Present the data from specimens where display_name is not null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE display_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens WHERE display_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "3eea55cd4532636ddffe_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "not_null", "question": "Find the rows in specimens where display_name is not null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE display_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens WHERE display_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "924dbd9ad819c0a58b24_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sum", "question": "inspections entity ke liye salary ka total output do.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT SUM(salary) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT SUM(salary) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "82c104614df1ce6e2762_hinglish", "language": "hinglish", "backend": "supabase", "operation": "gt", "question": "specimens se un rows ko display karo jahan salary 100 se more hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE salary > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE salary > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "35406fa024e66bcb5dd7_hinglish", "language": "hinglish", "backend": "supabase", "operation": "lt", "question": "specimens se un rows ko query karo jahan score 10000 se kam hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE score < 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE score < 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7094d70d08791284b632_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "min", "question": "What is the floor salary in reservations?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT MIN(salary) FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT MIN(salary) FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b30330bb2c17b32c6883_hi", "language": "hi", "backend": "supabase", "operation": "sort_asc", "question": "exhibits से सभी डेटा cost के बढ़ते हुए क्रम में लें और दिखाएं।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits ORDER BY cost ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY cost ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4a79981adf7647bac6b9_en", "language": "en", "backend": "supabase", "operation": "gt", "question": "Bring up the rows in exhibits where amount is over 500.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE amount > 500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE amount > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4fb8670235a435c82c88_hinglish", "language": "hinglish", "backend": "supabase", "operation": "describe", "question": "Mujhe exhibits ke columns aur types ke baare mein info do.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_exhibits", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "ae26d8175253b650bb6b_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "missing_value", "question": "execute filter on specimens for region, but i have not specified a value", "expected": {"action": "clarify", "question": "Which value should region equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should region equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "e8cfdfba296bc283222a_en", "language": "en", "backend": "supabase", "operation": "month", "question": "Select entries from exhibits where start_date corresponds to 7.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE EXTRACT(MONTH FROM start_date) = 7;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE EXTRACT(MONTH FROM start_date) = 7;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "71349134f8ae16809501_hi", "language": "hi", "backend": "supabase", "operation": "sort_desc", "question": "exhibits की पूरी सारणी salary के उच्चतम से निम्नतम क्रम में दिखाएँ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits ORDER BY salary DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5c6734bdfead38b51752_hinglish", "language": "hinglish", "backend": "supabase", "operation": "gte", "question": "specimens se rows dikhao jahan quantity 10000 ya usse bada hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE quantity >= 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE quantity >= 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "aeeb76897a32b9fd2df4_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "min", "question": "What is the floor amount in reservations?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT MIN(amount) FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT MIN(amount) FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "785d62b3d5e4932f5c33_en", "language": "en", "backend": "supabase", "operation": "bottom", "question": "Present the 20 minimal rating entries from inspections in ascending order.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections ORDER BY rating ASC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections ORDER BY rating ASC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "785d62b3d5e4932f5c33_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "bottom", "question": "print the 20 minimal rating rows in inspections from lowest up", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections ORDER BY rating ASC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections ORDER BY rating ASC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0b8689cf8e381cbcfe86_hi", "language": "hi", "backend": "supabase", "operation": "and", "question": "reservations में वे पंक्तियाँ खोजें जहाँ department pending है और price 10000 से ऊपर है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE department = 'pending' AND price > 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE department = 'pending' AND price > 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7094d70d08791284b632_hinglish", "language": "hinglish", "backend": "supabase", "operation": "min", "question": "reservations mein salary ka minimum reading kya hai?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT MIN(salary) FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT MIN(salary) FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8b1939fffee21884f667_en", "language": "en", "backend": "supabase", "operation": "null", "question": "I want to view the rows in specimens where item_name is null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE item_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "1896aafc31bd0a7d24ec_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sum", "question": "exhibits entity ke liye balance ka total output do.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT SUM(balance) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT SUM(balance) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b564e1a7b1e7fc6d2ab4_hi", "language": "hi", "backend": "supabase", "operation": "avg", "question": "specimens में score का औसत लौटाएँ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT AVG(score) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT AVG(score) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e92015bce168f8f9a90a_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sort_desc", "question": "retrieve all records from exhibits sorted by score high to low", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits ORDER BY score DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits ORDER BY score DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "223765dc6e033f0ab2fd_hinglish", "language": "hinglish", "backend": "supabase", "operation": "month", "question": "Show me the data from specimens where created_at ka month 9 hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE EXTRACT(MONTH FROM created_at) = 9;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT * FROM specimens WHERE EXTRACT(MONTH FROM created_at) = 9;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "4a79981adf7647bac6b9_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "gt", "question": "show me the entries in exhibits where amount is strictly greater than 500", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE amount > 500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE amount > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "61b3db48f1ff12dc69e1_en", "language": "en", "backend": "supabase", "operation": "group_sum", "question": "Please provide the total quantity for each kind in exhibits.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT kind, SUM(quantity) FROM exhibits GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT kind, SUM(quantity) FROM exhibits GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e8cfdfba296bc283222a_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "month", "question": "extract rows from exhibits where start_date month is 7", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE EXTRACT(MONTH FROM start_date) = 7;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE EXTRACT(MONTH FROM start_date) = 7;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8fa63f2c53b1bc742d49_en", "language": "en", "backend": "supabase", "operation": "null", "question": "I want to view the rows in inspections where title is null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "393d51bc9e30d68b474e_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "and", "question": "find rows in inspections where status is active and cost is above 5", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE status = 'active' AND cost > 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE status = 'active' AND cost > 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "3eea55cd4532636ddffe_hinglish", "language": "hinglish", "backend": "supabase", "operation": "not_null", "question": "specimens ka data present karo jahan display_name null nahi hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE display_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE display_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0ce22f4c8bc7554d586d_hi", "language": "hi", "backend": "supabase", "operation": "lt", "question": "reservations से उन पंक्तियों का क्वेरी करें जहाँ score 50 से कम है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE score < 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE score < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5ce4cc7ea9d93a0aa4c5_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "sum", "question": "i want the total quantity of reservations", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT SUM(quantity) FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT SUM(quantity) FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "aeeb76897a32b9fd2df4_hinglish", "language": "hinglish", "backend": "supabase", "operation": "min", "question": "reservations mein amount ka minimum reading kya hai?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT MIN(amount) FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT MIN(amount) FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "475287613764159ee5eb_en", "language": "en", "backend": "supabase", "operation": "group_count", "question": "Find the count of rows per category in reservations.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "475287613764159ee5eb_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "group_count", "question": "give me the count for each category rows in reservations", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "fd0bab1cc08d7298f131_hinglish", "language": "hinglish", "backend": "supabase", "operation": "top", "question": "Top 20 rows exhibits se lo, salary ke hisaab se highest first.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "61b3db48f1ff12dc69e1_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "group_sum", "question": "I need to see the total quantity for each kind in exhibits", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT kind, SUM(quantity) FROM exhibits GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT kind, SUM(quantity) FROM exhibits GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c86deada03a046468575_hinglish", "language": "hinglish", "backend": "supabase", "operation": "project", "question": "bas reservations ka item_name do please", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT item_name FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "864b7a7bf739071bb030_hinglish", "language": "hinglish", "backend": "supabase", "operation": "and", "question": "specimens mein wo rows dhundo jahan status Pune hai aur balance 0 se upar hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE status = 'Pune' AND balance > 0;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE status = 'Pune' AND balance > 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2ffce857234ca4c32ec9_en", "language": "en", "backend": "supabase", "operation": "project", "question": "Display title from reservations as the sole result.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT title FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT title FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7d18561ec942f18f1299_hi", "language": "hi", "backend": "supabase", "operation": "count_eq", "question": "reservations में status O'Reilly वाले रिकॉर्ड्स की संख्या दीजिए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM reservations WHERE status = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM reservations WHERE status = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "393d51bc9e30d68b474e_en", "language": "en", "backend": "supabase", "operation": "and", "question": "Show me the inspections rows where status is active and cost exceeds 5.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE status = 'active' AND cost > 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE status = 'active' AND cost > 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7a33de7fd7159808b04a_en", "language": "en", "backend": "supabase", "operation": "month", "question": "Select entries from inspections where created_at corresponds to 11.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE EXTRACT(MONTH FROM created_at) = 11;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE EXTRACT(MONTH FROM created_at) = 11;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e02d95c6aba4ed336a6a_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "gte", "question": "output records from reservations where cost is not below 50", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE cost >= 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE cost >= 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "40464b00088c9159e3d0_en", "language": "en", "backend": "supabase", "operation": "count", "question": "What is the volume of data rows in inspections?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT COUNT(*) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT COUNT(*) FROM inspections WHERE department = 'Agra';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "7094d70d08791284b632_en", "language": "en", "backend": "supabase", "operation": "min", "question": "What is the baseline salary value in reservations?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT MIN(salary) FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT MIN(salary) FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "475287613764159ee5eb_hinglish", "language": "hinglish", "backend": "supabase", "operation": "group_count", "question": "reservations me pratyek category ke liye rows ki sankhya dhundo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1695d78b68e4d8fca6f0_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "and", "question": "find rows in exhibits where region is Delhi and amount is above 1500", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE region = 'Delhi' AND amount > 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE region = 'Delhi' AND amount > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e92015bce168f8f9a90a_en", "language": "en", "backend": "supabase", "operation": "sort_desc", "question": "Display the entire table exhibits sorted by score from highest to lowest.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits ORDER BY score DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits ORDER BY score DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e92015bce168f8f9a90a_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sort_desc", "question": "exhibits ki poori table score highest se lowest order mein dikhao.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits ORDER BY score DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits ORDER BY score DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4a79981adf7647bac6b9_hinglish", "language": "hinglish", "backend": "supabase", "operation": "gt", "question": "exhibits se un rows ko display karo jahan amount 500 se more hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE amount > 500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE amount > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "645fd36c186709e64922_hi", "language": "hi", "backend": "supabase", "operation": "between", "question": "specimens से वे सभी आइटम चुनें जहाँ amount कॉलम का मान 1000 और 1010 के बीच है, समावेशी।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE amount BETWEEN 1000 AND 1010;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": false, "success": false, "error": "'query'"}} +{"id": "61b3db48f1ff12dc69e1_hinglish", "language": "hinglish", "backend": "supabase", "operation": "group_sum", "question": "exhibits me kind ke hisaab se quantity ka total do please.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT kind, SUM(quantity) FROM exhibits GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT kind, SUM(quantity) FROM exhibits GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9ebca064e63b0347236b_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "having", "question": "Tell me the region groups in specimens with more than 5 rows and their count values.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "9ebca064e63b0347236b_hinglish", "language": "hinglish", "backend": "supabase", "operation": "having", "question": "Please specimens me 5 se zyada rows wale region groups aur unki count ko display karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmens\",\"query\":\"SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "ae26d8175253b650bb6b_en", "language": "en", "backend": "supabase", "operation": "missing_value", "question": "Execute a filter on specimens based on region, but I have not specified a value.", "expected": {"action": "clarify", "question": "Which value should region equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should region equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "c86deada03a046468575_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "project", "question": "display item_name from reservations and nothing else", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT item_name FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b7703ebecadd9433c5f8_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "top", "question": "I need 3 rows from reservations with quantity sorted descending.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations ORDER BY quantity DESC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations ORDER BY quantity DESC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e8cfdfba296bc283222a_hinglish", "language": "hinglish", "backend": "supabase", "operation": "month", "question": "Show me the data from exhibits where start_date ka month 7 hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE EXTRACT(MONTH FROM start_date) = 7;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE EXTRACT(MONTH FROM start_date) = 7;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bf7bc588f62f075ec1ca_en", "language": "en", "backend": "supabase", "operation": "count", "question": "What is the volume of data rows in exhibits?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT COUNT(*) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT COUNT(*) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bd5fbea0dbbb3414c53c_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "having", "question": "Tell me the region groups in specimens with more than 0 rows and their count values.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 0;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmens\",\"query\":\"SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "bd5fbea0dbbb3414c53c_hinglish", "language": "hinglish", "backend": "supabase", "operation": "having", "question": "Please specimens me 0 se zyada rows wale region groups aur unki count ko display karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 0;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmens\",\"query\":\"SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "9bb888f912ca90f30c9a_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "top", "question": "I need 20 rows from reservations with rating sorted descending.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations ORDER BY rating DESC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations ORDER BY rating DESC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a7efbaf306821c7ab040_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "year", "question": "Fetch reservations entries where order_date year is 2023.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE EXTRACT(YEAR FROM order_date) = 2023;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE EXTRACT(YEAR FROM order_date) = 2023;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a7efbaf306821c7ab040_hinglish", "language": "hinglish", "backend": "supabase", "operation": "year", "question": "reservations mein order_date mein 2023 wale records dikhao.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE EXTRACT(YEAR FROM order_date) = 2023;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE EXTRACT(YEAR FROM order_date) = 2023;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7a33de7fd7159808b04a_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "month", "question": "extract rows from inspections where created_at month is 11", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE EXTRACT(MONTH FROM created_at) = 11;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE EXTRACT(MONTH FROM created_at) = 11;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5ce4cc7ea9d93a0aa4c5_en", "language": "en", "backend": "supabase", "operation": "sum", "question": "Output the total quantity for the reservations entity.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT SUM(quantity) FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT SUM(quantity) FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "aeeb76897a32b9fd2df4_en", "language": "en", "backend": "supabase", "operation": "min", "question": "What is the baseline amount value in reservations?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT MIN(amount) FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT MIN(amount) FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7d4234476674a04380e6_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "gte", "question": "output records from reservations where score is not below 1500", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE score >= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE score >= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1695d78b68e4d8fca6f0_en", "language": "en", "backend": "supabase", "operation": "and", "question": "Show me the exhibits rows where region is Delhi and amount exceeds 1500.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE region = 'Delhi' AND amount > 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE region = 'Delhi' AND amount > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "fd0bab1cc08d7298f131_en", "language": "en", "backend": "supabase", "operation": "top", "question": "Give me the top 20 rows of exhibits sorted by salary from high to low.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "039280d3483e998ccd1b_en", "language": "en", "backend": "supabase", "operation": "group_count", "question": "Find the count of rows per department in reservations.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT department, COUNT(*) FROM reservations GROUP BY department;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT department, COUNT(*) FROM reservations GROUP BY department;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "039280d3483e998ccd1b_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "group_count", "question": "give me the count for each department rows in reservations", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT department, COUNT(*) FROM reservations GROUP BY department;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT department, COUNT(*) FROM reservations GROUP BY department;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "523a83d6ca07c1a1334a_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "missing_value", "question": "execute filter on exhibits for department, but i have not specified a value", "expected": {"action": "clarify", "question": "Which value should department equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should department equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "5d05a2e11d5abab8423e_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "missing_value", "question": "execute filter on inspections for region, but i have not specified a value", "expected": {"action": "clarify", "question": "Which value should region equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should region equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "f4fb7fafb95b5c8287f5_en", "language": "en", "backend": "supabase", "operation": "group_count", "question": "Find the count of rows per department in reservations.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT department, COUNT(*) FROM reservations GROUP BY department;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT department, COUNT(*) FROM reservations GROUP BY department;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f4fb7fafb95b5c8287f5_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "group_count", "question": "give me the count for each department rows in reservations", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT department, COUNT(*) FROM reservations GROUP BY department;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT department, COUNT(*) FROM reservations GROUP BY department;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a7efbaf306821c7ab040_en", "language": "en", "backend": "supabase", "operation": "year", "question": "Show me the reservations rows for 2023 using order_date.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE EXTRACT(YEAR FROM order_date) = 2023;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE EXTRACT(YEAR FROM order_date) = 2023;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e02d95c6aba4ed336a6a_en", "language": "en", "backend": "supabase", "operation": "gte", "question": "Show the data from reservations where cost is 50 or above.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE cost >= 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE cost >= 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e02d95c6aba4ed336a6a_hinglish", "language": "hinglish", "backend": "supabase", "operation": "gte", "question": "reservations se rows dikhao jahan cost 50 ya usse bada hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE cost >= 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE cost >= 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5ce4cc7ea9d93a0aa4c5_hinglish", "language": "hinglish", "backend": "supabase", "operation": "sum", "question": "reservations entity ke liye quantity ka total output do.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT SUM(quantity) FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT SUM(quantity) FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "039280d3483e998ccd1b_hinglish", "language": "hinglish", "backend": "supabase", "operation": "group_count", "question": "reservations me pratyek department ke liye rows ki sankhya dhundo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT department, COUNT(*) FROM reservations GROUP BY department;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT department, COUNT(*) FROM reservations GROUP BY department;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "393d51bc9e30d68b474e_hinglish", "language": "hinglish", "backend": "supabase", "operation": "and", "question": "inspections mein wo rows dhundo jahan status active hai aur cost 5 se upar hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE status = 'active' AND cost > 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE status = 'active' AND cost > 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f4fb7fafb95b5c8287f5_hinglish", "language": "hinglish", "backend": "supabase", "operation": "group_count", "question": "reservations me pratyek department ke liye rows ki sankhya dhundo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT department, COUNT(*) FROM reservations GROUP BY department;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT department, COUNT(*) FROM reservations GROUP BY department;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7a33de7fd7159808b04a_hinglish", "language": "hinglish", "backend": "supabase", "operation": "month", "question": "Show me the data from inspections where created_at ka month 11 hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE EXTRACT(MONTH FROM created_at) = 11;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE EXTRACT(MONTH FROM created_at) = 11;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ac6cc6f05e6d013971e5_hi", "language": "hi", "backend": "supabase", "operation": "max", "question": "specimens में quantity का सबसे बड़ा नंबर क्या है?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT MAX(quantity) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT MAX(quantity) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1a1af4deb475f0224a59_hi", "language": "hi", "backend": "supabase", "operation": "bottom", "question": "price के बढ़ते क्रम में inspections की 10 सबसे कम पंक्तियां, सबसे कम से शुरू होकर, लें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections ORDER BY price ASC LIMIT 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY price ASC LIMIT 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1d82ef2b9527e797c466_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "date_after", "question": "Query rows in inspections where order_date is 2022-04-15 onwards.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE order_date >= '2022-04-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE order_date >= '2022-04-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7d4234476674a04380e6_en", "language": "en", "backend": "supabase", "operation": "gte", "question": "Show the data from reservations where score is 1500 or above.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE score >= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE score >= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7d4234476674a04380e6_hinglish", "language": "hinglish", "backend": "supabase", "operation": "gte", "question": "reservations se rows dikhao jahan score 1500 ya usse bada hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE score >= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE score >= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c86deada03a046468575_en", "language": "en", "backend": "supabase", "operation": "project", "question": "Display item_name from reservations as the sole result.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT item_name FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b7703ebecadd9433c5f8_hinglish", "language": "hinglish", "backend": "supabase", "operation": "top", "question": "Top 3 rows reservations se lo, quantity ke hisaab se highest first.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations ORDER BY quantity DESC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations ORDER BY quantity DESC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f1f8414c2d47f5396d69_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "date_after", "question": "Query rows in exhibits where created_at is 2023-03-15 onwards.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE created_at >= '2023-03-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE created_at >= '2023-03-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9bb888f912ca90f30c9a_hinglish", "language": "hinglish", "backend": "supabase", "operation": "top", "question": "Top 20 rows reservations se lo, rating ke hisaab se highest first.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations ORDER BY rating DESC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations ORDER BY rating DESC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "dd6cdefd15edcdcddb76_hi", "language": "hi", "backend": "supabase", "operation": "bottom", "question": "salary के बढ़ते क्रम में exhibits की 5 सबसे कम पंक्तियां, सबसे कम से शुरू होकर, लें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits ORDER BY salary ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits ORDER BY salary ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "dddd09f34c0e849282a8_en", "language": "en", "backend": "supabase", "operation": "gt", "question": "Bring up the rows in reservations where salary is over 20.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE salary > 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE salary > 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f35f7a78a14e903920ee_hi", "language": "hi", "backend": "supabase", "operation": "month", "question": "मुझे reservations से वे डेटा पंक्तियां दें जहाँ order_date का महीना 3 है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE EXTRACT(MONTH FROM order_date) = 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE EXTRACT(MONTH FROM order_date) = 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1695d78b68e4d8fca6f0_hinglish", "language": "hinglish", "backend": "supabase", "operation": "and", "question": "exhibits mein wo rows dhundo jahan region Delhi hai aur amount 1500 se upar hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE region = 'Delhi' AND amount > 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE region = 'Delhi' AND amount > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c07bcd68edd3354fcd83_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "all", "question": "fetch all rows from specimens", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ae26d8175253b650bb6b_hinglish", "language": "hinglish", "backend": "supabase", "operation": "missing_value", "question": "region ke basis pe specimens pe ek filter execute karo, lekin maine koi value specify nahi ki hai.", "expected": {"action": "clarify", "question": "region ki kaunsi value chahiye?"}, "output": "{\"action\":\"clarify\",\"question\":\"region ki kaunsi value chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "375b86b5cd0d00cd7868_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "having", "question": "Tell me the department groups in exhibits with more than 2 rows and their count values.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "375b86b5cd0d00cd7868_hinglish", "language": "hinglish", "backend": "supabase", "operation": "having", "question": "Please exhibits me 2 se zyada rows wale department groups aur unki count ko display karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "523a83d6ca07c1a1334a_en", "language": "en", "backend": "supabase", "operation": "missing_value", "question": "Execute a filter on exhibits based on department, but I have not specified a value.", "expected": {"action": "clarify", "question": "Which value should department equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should department equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "db7eb47a2bad512e837d_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "all", "question": "fetch all rows from specimens", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2ffce857234ca4c32ec9_hi", "language": "hi", "backend": "supabase", "operation": "project", "question": "सिर्फ reservations का title दिखाइए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT title FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT title FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ccbb8ac5e43114f43047_en", "language": "en", "backend": "supabase", "operation": "between", "question": "Select all items from exhibits where the balance column has a value between 50 and 100, inclusive.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE balance BETWEEN 50 AND 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE balance BETWEEN 50 AND 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "28530a7607feca135b73_hi", "language": "hi", "backend": "supabase", "operation": "sum", "question": "specimens में price के सभी मानों का योग करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT SUM(price) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT SUM(price) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5d05a2e11d5abab8423e_en", "language": "en", "backend": "supabase", "operation": "missing_value", "question": "Execute a filter on inspections based on region, but I have not specified a value.", "expected": {"action": "clarify", "question": "Which value should region equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should region equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "6dd9f03c26298630777f_en", "language": "en", "backend": "supabase", "operation": "between", "question": "Select all items from inspections where the score column has a value between 1 and 101, inclusive.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE score BETWEEN 1 AND 101;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE score BETWEEN 1 AND 101;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6c37ae504c7cb31a6ef3_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "list_tables", "question": "tell me the tables in this database", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "dddd09f34c0e849282a8_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "gt", "question": "show me the entries in reservations where salary is strictly greater than 20", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE salary > 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE salary > 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a4cdc86496ddd0c1cc90_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "avg", "question": "determine the mean cost in specimens", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT AVG(cost) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT AVG(cost) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "785d62b3d5e4932f5c33_hinglish", "language": "hinglish", "backend": "supabase", "operation": "bottom", "question": "rating ke badhte order me inspections ki 20 sabse kam rows, lowest se shuru karke, lao.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections ORDER BY rating ASC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections ORDER BY rating ASC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "23c0aa3a4596f995ad2a_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "count", "question": "how many lines in specimens?", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "58eb6ab2fb9c69becdbd_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "date_after", "question": "Query rows in reservations where created_at is 2022-08-15 onwards.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE created_at >= '2022-08-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE created_at >= '2022-08-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c07bcd68edd3354fcd83_en", "language": "en", "backend": "mysql", "operation": "all", "question": "Get all rows from specimens.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9ebca064e63b0347236b_en", "language": "en", "backend": "supabase", "operation": "having", "question": "Give me the region groups in specimens where the row count is higher than 5, along with the count values.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "e3d12f1bf51bbf54a331_hi", "language": "hi", "backend": "supabase", "operation": "contains", "question": "specimens से वे पंक्तियाँ दिखाएं जिनमें description North के समान है, मामला अंतर नजरअंदाज करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE LOWER(description) LIKE LOWER('%North%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE LOWER(description) LIKE LOWER('%North%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "55cd623d82474692b1a9_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "count", "question": "how many lines in specimens?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1110104f12e10c5a007a_hi", "language": "hi", "backend": "supabase", "operation": "sum", "question": "inspections में amount के सभी मानों का योग करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT SUM(amount) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT SUM(amount) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b7703ebecadd9433c5f8_en", "language": "en", "backend": "supabase", "operation": "top", "question": "Give me the top 3 rows of reservations sorted by quantity from high to low.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations ORDER BY quantity DESC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations ORDER BY quantity DESC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "db7eb47a2bad512e837d_en", "language": "en", "backend": "mysql", "operation": "all", "question": "Get all rows from specimens.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bd5fbea0dbbb3414c53c_en", "language": "en", "backend": "supabase", "operation": "having", "question": "Give me the region groups in specimens where the row count is higher than 0, along with the count values.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 0;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "e6380b8531b4b3eb7e38_hinglish", "language": "hinglish", "backend": "mysql", "operation": "project", "question": "bas specimens ka name do please", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT name FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "924dbd9ad819c0a58b24_hi", "language": "hi", "backend": "supabase", "operation": "sum", "question": "inspections में salary के सभी मानों का योग करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT SUM(salary) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT SUM(salary) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9bb888f912ca90f30c9a_en", "language": "en", "backend": "supabase", "operation": "top", "question": "Give me the top 20 rows of reservations sorted by rating from high to low.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations ORDER BY rating DESC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations ORDER BY rating DESC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6c37ae504c7cb31a6ef3_en", "language": "en", "backend": "mysql", "operation": "list_tables", "question": "Display the schema tables in this database.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "6c37ae504c7cb31a6ef3_hinglish", "language": "hinglish", "backend": "mysql", "operation": "list_tables", "question": "Is database ki tables ka detail do.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "dddd09f34c0e849282a8_hinglish", "language": "hinglish", "backend": "supabase", "operation": "gt", "question": "reservations se un rows ko display karo jahan salary 20 se more hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE salary > 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE salary > 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "40464b00088c9159e3d0_hi", "language": "hi", "backend": "supabase", "operation": "count", "question": "inspections में कितनी लाइनें मौजूद हैं?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT COUNT(*) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT COUNT(*) FROM inspections WHERE department = 'Agra';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "8efcf517aee64807a8e5_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "avg", "question": "determine the mean balance in specimens", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(balance) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "797b9b914530f2eb502a_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "final", "question": "db tool returned 48 rows. tell me the total.", "expected": {"action": "answer", "text": "The tool returned 48 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 48 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "797b9b914530f2eb502a_hinglish", "language": "hinglish", "backend": "supabase", "operation": "final", "question": "Kitni rows aayin database tool se? Count hai 48.", "expected": {"action": "answer", "text": "Tool ne 48 rows return ki."}, "output": "{\"action\":\"answer\",\"text\":\"Tool ne 48 rows return ki.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "23c0aa3a4596f995ad2a_hinglish", "language": "hinglish", "backend": "mysql", "operation": "count", "question": "Batao specimens mein kitni entries hain.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "49c2e66e48d062e5c69c_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "write", "question": "Remove all records from specimens now.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"clarify\",\"question\":\"This connection is read-only. Would you like a SELECT query instead?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "64372a29664a7eaddbe2_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "avg", "question": "determine the mean amount in specimens", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT AVG(amount) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT AVG(amount) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "55cd623d82474692b1a9_hinglish", "language": "hinglish", "backend": "mysql", "operation": "count", "question": "Batao specimens mein kitni entries hain.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e02df083b1ff1b7fbcf8_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "join_filter", "question": "show me exhibits.customer_name for exhibits entries where exhibits.group_id links to exhibits_groups.id and exhibits_groups.label is North", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT exhibits.customer_name FROM exhibits JOIN exhibits_groups ON exhibits.group_id = exhibits_groups.id WHERE exhibits_groups.label = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT exhibits.customer_name FROM exhibits JOIN exhibits_groups ON exhibits.group_id = exhibits_groups.id WHERE exhibits_groups.label = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e02df083b1ff1b7fbcf8_hinglish", "language": "hinglish", "backend": "supabase", "operation": "join_filter", "question": "Please exhibits.customer_name dikhao jahan exhibits exhibits.group_id ke through exhibits_groups se juda hai aur exhibits_groups.label North hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT exhibits.customer_name FROM exhibits JOIN exhibits_groups ON exhibits.group_id = exhibits_groups.id WHERE exhibits_groups.label = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT exhibits.customer_name FROM exhibits JOIN exhibits_groups ON exhibits.group_id = exhibits_groups.id WHERE exhibits_groups.label = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2ca6667f987742cde1e7_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "write", "question": "Remove all records from specimens now.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"clarify\",\"question\":\"This connection is read-only. Would you like a SELECT query instead?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "ccbb8ac5e43114f43047_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "between", "question": "pull the rows from exhibits where balance is greater than or equal to 50 and less than or equal to 100", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE balance BETWEEN 50 AND 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE balance BETWEEN 50 AND 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bf7bc588f62f075ec1ca_hi", "language": "hi", "backend": "supabase", "operation": "count", "question": "exhibits में कितनी लाइनें मौजूद हैं?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT COUNT(*) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT COUNT(*) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9bf8948becf74e81a103_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "list_tables", "question": "tell me the tables in this database", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "e6380b8531b4b3eb7e38_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "project", "question": "display name from specimens and nothing else", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT name FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6dd9f03c26298630777f_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "between", "question": "pull the rows from inspections where score is greater than or equal to 1 and less than or equal to 101", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE score BETWEEN 1 AND 101;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE score BETWEEN 1 AND 101;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a4cdc86496ddd0c1cc90_en", "language": "en", "backend": "mysql", "operation": "avg", "question": "Return the average cost in specimens.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT AVG(cost) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT AVG(cost) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1d82ef2b9527e797c466_en", "language": "en", "backend": "supabase", "operation": "date_after", "question": "Extract rows from inspections where order_date falls on 2022-04-15 or beyond.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE order_date >= '2022-04-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE order_date >= '2022-04-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c4210296305105ab7472_en", "language": "en", "backend": "mysql", "operation": "ambiguous", "question": "Print the best available records in specimens.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "c4210296305105ab7472_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "ambiguous", "question": "Give me the top records in specimens.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "1896aafc31bd0a7d24ec_hi", "language": "hi", "backend": "supabase", "operation": "sum", "question": "exhibits में balance के सभी मानों का योग करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT SUM(balance) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT SUM(balance) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8a3194b9f492b50b8100_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "distinct", "question": "i need the distinct category values from specimens", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT DISTINCT category FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT category FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d61a27de98bb7d44f15d_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "count", "question": "how many lines in specimens?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "3b3a8e441f0ae5cbaad4_en", "language": "en", "backend": "mysql", "operation": "ambiguous", "question": "Print the best available records in specimens.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "3b3a8e441f0ae5cbaad4_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "ambiguous", "question": "Give me the top records in specimens.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "69ac9800bd7a3ee23ccc_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "distinct", "question": "i need the distinct category values from specimens", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT DISTINCT category FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT category FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f1f8414c2d47f5396d69_en", "language": "en", "backend": "supabase", "operation": "date_after", "question": "Extract rows from exhibits where created_at falls on 2023-03-15 or beyond.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE created_at >= '2023-03-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE created_at >= '2023-03-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e02df083b1ff1b7fbcf8_en", "language": "en", "backend": "supabase", "operation": "join_filter", "question": "Give me exhibits.customer_name for exhibits entries where exhibits.group_id points to exhibits_groups.id and exhibits_groups.label is North.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT exhibits.customer_name FROM exhibits JOIN exhibits_groups ON exhibits.group_id = exhibits_groups.id WHERE exhibits_groups.label = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT exhibits.customer_name FROM exhibits JOIN exhibits_groups ON exhibits.group_id = exhibits_groups.id WHERE exhibits_groups.label = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ccbb8ac5e43114f43047_hinglish", "language": "hinglish", "backend": "supabase", "operation": "between", "question": "exhibits se wo saare items select karo jahan balance column ka value 50 aur 100 ke beech hai, inclusive.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE balance BETWEEN 50 AND 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE balance BETWEEN 50 AND 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8fa63f2c53b1bc742d49_hi", "language": "hi", "backend": "supabase", "operation": "null", "question": "inspections से title null वाली पंक्तियाँ चुनें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9bf8948becf74e81a103_en", "language": "en", "backend": "mysql", "operation": "list_tables", "question": "Display the schema tables in this database.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "9bf8948becf74e81a103_hinglish", "language": "hinglish", "backend": "mysql", "operation": "list_tables", "question": "Is database ki tables ka detail do.", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "6dd9f03c26298630777f_hinglish", "language": "hinglish", "backend": "supabase", "operation": "between", "question": "inspections se wo saare items select karo jahan score column ka value 1 aur 101 ke beech hai, inclusive.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE score BETWEEN 1 AND 101;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE score BETWEEN 1 AND 101;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1523073b9b419fc60ded_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "count_eq", "question": "how many specimens rows where city is Books", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT COUNT(*) FROM specimens WHERE city = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM specimens WHERE city = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a4cdc86496ddd0c1cc90_hinglish", "language": "hinglish", "backend": "mysql", "operation": "avg", "question": "specimens me cost ka average return karo.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT AVG(cost) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT AVG(cost) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "41ea5651dda516a4ac30_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "date_after", "question": "Query rows in reservations where order_date is 2023-12-15 onwards.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE order_date >= '2023-12-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE order_date >= '2023-12-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "adff4eabe2639c0c128c_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "all", "question": "fetch all rows from inspections", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8efcf517aee64807a8e5_en", "language": "en", "backend": "mysql", "operation": "avg", "question": "Return the average balance in specimens.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(balance) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8b1939fffee21884f667_hi", "language": "hi", "backend": "supabase", "operation": "null", "question": "specimens से item_name null वाली पंक्तियाँ चुनें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE item_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "51f14088477a5f6be9c5_hi", "language": "hi", "backend": "supabase", "operation": "describe", "question": "टेबल specimens के कॉलम संरचना और टाइप्स दिखाइए।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "c07bcd68edd3354fcd83_hinglish", "language": "hinglish", "backend": "mysql", "operation": "all", "question": "specimens ke saare records get karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d61a27de98bb7d44f15d_hinglish", "language": "hinglish", "backend": "mysql", "operation": "count", "question": "Batao specimens mein kitni entries hain.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "64372a29664a7eaddbe2_en", "language": "en", "backend": "mysql", "operation": "avg", "question": "Return the average amount in specimens.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT AVG(amount) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT AVG(amount) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6d9e81a9f52b8d33060a_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "all", "question": "fetch all rows from exhibits", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c86deada03a046468575_hi", "language": "hi", "backend": "supabase", "operation": "project", "question": "सिर्फ reservations का item_name दिखाइए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT item_name FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "940596942f7c5e4bf758_hi", "language": "hi", "backend": "supabase", "operation": "list_tables", "question": "इस डेटाबेस की तालिकाओं का विवरण दीजिए।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "523a83d6ca07c1a1334a_hinglish", "language": "hinglish", "backend": "supabase", "operation": "missing_value", "question": "department ke basis pe exhibits pe ek filter execute karo, lekin maine koi value specify nahi ki hai.", "expected": {"action": "clarify", "question": "department ki kaunsi value chahiye?"}, "output": "{\"action\":\"clarify\",\"question\":\"department ki kaunsi value chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "db7eb47a2bad512e837d_hinglish", "language": "hinglish", "backend": "mysql", "operation": "all", "question": "specimens ke saare records get karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5d05a2e11d5abab8423e_hinglish", "language": "hinglish", "backend": "supabase", "operation": "missing_value", "question": "region ke basis pe inspections pe ek filter execute karo, lekin maine koi value specify nahi ki hai.", "expected": {"action": "clarify", "question": "region ki kaunsi value chahiye?"}, "output": "{\"action\":\"clarify\",\"question\":\"region ki kaunsi value chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "3eea55cd4532636ddffe_hi", "language": "hi", "backend": "supabase", "operation": "not_null", "question": "specimens का डेटा दिखाएं जहाँ display_name null नहीं है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE display_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE display_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "adff4eabe2639c0c128c_en", "language": "en", "backend": "mysql", "operation": "all", "question": "Get all rows from inspections.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8efcf517aee64807a8e5_hinglish", "language": "hinglish", "backend": "mysql", "operation": "avg", "question": "specimens me balance ka average return karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(balance) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "cd5bb3382b9e6c8604e8_en", "language": "en", "backend": "mysql", "operation": "max", "question": "What is the largest number for rating in specimens?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MAX(rating) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(rating) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "cd5bb3382b9e6c8604e8_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "max", "question": "what is the upper bound of rating in specimens?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MAX(rating) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(rating) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c4210296305105ab7472_hinglish", "language": "hinglish", "backend": "mysql", "operation": "ambiguous", "question": "specimens mein available best records print karo.", "expected": {"action": "clarify", "question": "Best ka matlab kya hai? Kaunsa column aur order?"}, "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "797b9b914530f2eb502a_en", "language": "en", "backend": "supabase", "operation": "final", "question": "The database tool returned 48 rows. Can you tell me the number?", "expected": {"action": "answer", "text": "The tool returned 48 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 48 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "9aed1992a793fbf97601_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "min", "question": "What is the floor price in exhibits?", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT MIN(price) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT MIN(price) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "58eb6ab2fb9c69becdbd_en", "language": "en", "backend": "supabase", "operation": "date_after", "question": "Extract rows from reservations where created_at falls on 2022-08-15 or beyond.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE created_at >= '2022-08-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE created_at >= '2022-08-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "49c2e66e48d062e5c69c_en", "language": "en", "backend": "mysql", "operation": "write", "question": "Expunge every row from specimens.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": false, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false}} +{"id": "64372a29664a7eaddbe2_hinglish", "language": "hinglish", "backend": "mysql", "operation": "avg", "question": "specimens me amount ka average return karo.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT AVG(amount) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT AVG(amount) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "3b3a8e441f0ae5cbaad4_hinglish", "language": "hinglish", "backend": "mysql", "operation": "ambiguous", "question": "specimens mein available best records print karo.", "expected": {"action": "clarify", "question": "Best ka matlab kya hai? Kaunsa column aur order?"}, "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "6d9e81a9f52b8d33060a_en", "language": "en", "backend": "mysql", "operation": "all", "question": "Get all rows from exhibits.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e2b48c4ec5252934cb20_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "count_eq", "question": "how many inspections rows where city is pending", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT COUNT(*) FROM inspections WHERE city = 'pending';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM inspections WHERE city = 'pending';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "375b86b5cd0d00cd7868_en", "language": "en", "backend": "supabase", "operation": "having", "question": "Give me the department groups in exhibits where the row count is higher than 2, along with the count values.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4745f53221e07d4f5ae1_hi", "language": "hi", "backend": "supabase", "operation": "contains", "question": "specimens से वे पंक्तियाँ दिखाएं जिनमें display_name Pune के समान है, मामला अंतर नजरअंदाज करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE LOWER(display_name) LIKE LOWER('%Pune%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE LOWER(display_name) LIKE LOWER('%Pune%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2ca6667f987742cde1e7_en", "language": "en", "backend": "mysql", "operation": "write", "question": "Expunge every row from specimens.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"clarify\",\"question\":\"This connection is read-only. Would you like a SELECT query instead?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "3fdbfd3a4d0df901f143_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "gte", "question": "output records from specimens where amount is not below 200", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM specimens WHERE amount >= 200;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE amount >= 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e114d1f216088c4302b2_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "distinct", "question": "i need the distinct department values from specimens", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT DISTINCT department FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT DISTINCT department FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e6380b8531b4b3eb7e38_en", "language": "en", "backend": "mysql", "operation": "project", "question": "Display name from specimens as the sole result.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT name FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "253e89e4cdbbd5c2d0f9_en", "language": "en", "backend": "mysql", "operation": "group_count", "question": "Find the count of rows per status in specimens.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT status, COUNT(*) FROM specimens GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT status, COUNT(*) FROM specimens GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "253e89e4cdbbd5c2d0f9_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "group_count", "question": "give me the count for each status rows in specimens", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT status, COUNT(*) FROM specimens GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT status, COUNT(*) FROM specimens GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0abe13c841fd7c3ac885_hi", "language": "hi", "backend": "supabase", "operation": "having", "question": "कृपया inspections में 5000 से अधिक पंक्तियों वाले region समूहों और उनकी गणना को प्रदर्शित करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT region, COUNT(*) FROM inspections GROUP BY region HAVING COUNT(*) > 5000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT region, COUNT(*) FROM inspections GROUP BY region HAVING COUNT(*) > 5000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e7d754ed36b247db55d3_en", "language": "en", "backend": "mysql", "operation": "not_null", "question": "Present the data from specimens where description is not null.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens WHERE description IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE description IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e7d754ed36b247db55d3_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "not_null", "question": "Find the rows in specimens where description is not null.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens WHERE description IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE description IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5a8f2e207d0efe5b018c_hi", "language": "hi", "backend": "supabase", "operation": "list_tables", "question": "इस डेटाबेस की तालिकाओं का विवरण दीजिए।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "49c2e66e48d062e5c69c_hinglish", "language": "hinglish", "backend": "mysql", "operation": "write", "question": "specimens ka pura content hatwa do.", "expected": {"action": "clarify", "question": "Yeh connection read-only hai. Kya SELECT query chahiye?"}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": false, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false}} +{"id": "d30264bbc3b08aedbc51_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sum", "question": "i want the total score of exhibits", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT SUM(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT SUM(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "aa4f0e1f29573e237ee1_hi", "language": "hi", "backend": "supabase", "operation": "list_tables", "question": "इस डेटाबेस की तालिकाओं का विवरण दीजिए।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_specimens", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "4fb8670235a435c82c88_hi", "language": "hi", "backend": "supabase", "operation": "describe", "question": "टेबल exhibits के कॉलम संरचना और टाइप्स दिखाइए।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"project_id": "demo_exhibits", "schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "8ebc9e868ee558a8ddfe_hi", "language": "hi", "backend": "supabase", "operation": "ambiguous", "question": "specimens में उपलब्ध सर्वोत्तम रिकॉर्ड प्रिंट करें।", "expected": {"action": "clarify", "question": "सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?"}, "output": "{\"action\":\"clarify\",\"question\":\"सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "2ca6667f987742cde1e7_hinglish", "language": "hinglish", "backend": "mysql", "operation": "write", "question": "specimens ka pura content hatwa do.", "expected": {"action": "clarify", "question": "Yeh connection read-only hai. Kya SELECT query chahiye?"}, "output": "{\"action\":\"clarify\",\"question\":\"Yeh connection read-only hai. Kya SELECT query chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "aa867c871299c26d5402_en", "language": "en", "backend": "mysql", "operation": "lt", "question": "Query specimens for rows where rating is less than 1000.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE rating < 1000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE rating < 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4c5f91f7793bd0200f31_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "top", "question": "I need 1 rows from specimens with amount sorted descending.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM specimens ORDER BY amount DESC LIMIT 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY amount DESC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "eab95ae98810f2228ba6_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "year", "question": "Fetch specimens entries where start_date year is 2026.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM specimens WHERE YEAR(start_date) = 2026;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE YEAR(start_date) = 2026;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d0346c74f1cfe556b2cb_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "project_eq", "question": "give me description from inspections where status is Books", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT description FROM inspections WHERE status = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT description FROM inspections WHERE status = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1523073b9b419fc60ded_en", "language": "en", "backend": "mysql", "operation": "count_eq", "question": "Provide the row count for specimens where city is Books.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT COUNT(*) FROM specimens WHERE city = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM specimens WHERE city = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1523073b9b419fc60ded_hinglish", "language": "hinglish", "backend": "mysql", "operation": "count_eq", "question": "Please count the rows in specimens where city is Books.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT COUNT(*) FROM specimens WHERE city = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM specimens WHERE city = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "16fbb58e3a07a1240f50_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "avg", "question": "determine the mean score in exhibits", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT AVG(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT AVG(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e71814820a36875f0ee5_en", "language": "en", "backend": "mysql", "operation": "not_null", "question": "Present the data from specimens where description is not null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM specimens WHERE description IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE description IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e71814820a36875f0ee5_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "not_null", "question": "Find the rows in specimens where description is not null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM specimens WHERE description IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE description IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5ce4cc7ea9d93a0aa4c5_hi", "language": "hi", "backend": "supabase", "operation": "sum", "question": "reservations में quantity के सभी मानों का योग करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT SUM(quantity) FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT SUM(quantity) FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "adff4eabe2639c0c128c_hinglish", "language": "hinglish", "backend": "mysql", "operation": "all", "question": "inspections ke saare records get karo.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "cd5bb3382b9e6c8604e8_hinglish", "language": "hinglish", "backend": "mysql", "operation": "max", "question": "specimens mein rating ka sabse bada number kya hai?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MAX(rating) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(rating) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7094d70d08791284b632_hi", "language": "hi", "backend": "supabase", "operation": "min", "question": "reservations में salary का न्यूनतम परिमाण क्या है?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT MIN(salary) FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT MIN(salary) FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1d82ef2b9527e797c466_hinglish", "language": "hinglish", "backend": "supabase", "operation": "date_after", "question": "inspections mein se wo rows chuno jinke order_date mein 2022-04-15 ya uske baad ki date hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE order_date >= '2022-04-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE order_date >= '2022-04-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "26db633822b8319f83ec_hi", "language": "hi", "backend": "supabase", "operation": "contains", "question": "reservations से वे पंक्तियाँ दिखाएं जिनमें name active के समान है, मामला अंतर नजरअंदाज करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE LOWER(name) LIKE LOWER('%active%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE LOWER(name) LIKE LOWER('%active%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "84b16996198346dc9882_hi", "language": "hi", "backend": "mysql", "operation": "missing_schema", "question": "कृपया exhibits के लिए SQL लिखने से पहले उपलब्ध तालिकाओं का पता लगाएं, क्योंकि मैंने डेटाबेस स्कीमा नहीं दी है।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "81b12886ddb859bb71db_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "count_eq", "question": "how many inspections rows where city is pending", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM inspections WHERE city = 'pending';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM inspections WHERE city = 'pending';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9aed1992a793fbf97601_hinglish", "language": "hinglish", "backend": "mysql", "operation": "min", "question": "exhibits mein price ka minimum reading kya hai?", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT MIN(price) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT MIN(price) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8a3194b9f492b50b8100_en", "language": "en", "backend": "mysql", "operation": "distinct", "question": "I want to see the distinct category values available in specimens.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT DISTINCT category FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT category FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8a3194b9f492b50b8100_hinglish", "language": "hinglish", "backend": "mysql", "operation": "distinct", "question": "Mujhe specimens mein available category ke distinct values dekhne hain.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT DISTINCT category FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT category FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2164b376f732a1c371ab_en", "language": "en", "backend": "mysql", "operation": "group_count", "question": "Find the count of rows per kind in specimens.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT kind, COUNT(*) FROM specimens GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM specimens GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2164b376f732a1c371ab_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "group_count", "question": "give me the count for each kind rows in specimens", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT kind, COUNT(*) FROM specimens GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM specimens GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ed474bd8a459cbfaa5d2_hi", "language": "hi", "backend": "supabase", "operation": "year", "question": "specimens में recorded_at में 2021 वाले रिकॉर्ड दिखाएं।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE EXTRACT(YEAR FROM recorded_at) = 2021;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE EXTRACT(YEAR FROM recorded_at) = 2021;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c0b9cf3b44d96839f256_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "min", "question": "What is the floor balance in inspections?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MIN(balance) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(balance) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a29fdee3339da6711080_en", "language": "en", "backend": "mysql", "operation": "bottom", "question": "Present the 20 minimal rating entries from specimens in ascending order.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens ORDER BY rating ASC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY rating ASC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a29fdee3339da6711080_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "bottom", "question": "print the 20 minimal rating rows in specimens from lowest up", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens ORDER BY rating ASC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY rating ASC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "61b3db48f1ff12dc69e1_hi", "language": "hi", "backend": "supabase", "operation": "group_sum", "question": "exhibits में kind के अनुसार quantity का कुल कृपया दें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT kind, SUM(quantity) FROM exhibits GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT kind, SUM(quantity) FROM exhibits GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6d9e81a9f52b8d33060a_hinglish", "language": "hinglish", "backend": "mysql", "operation": "all", "question": "exhibits ke saare records get karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bc8593644a72fe8b9c78_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "min", "question": "What is the floor score in exhibits?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MIN(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "69ac9800bd7a3ee23ccc_en", "language": "en", "backend": "mysql", "operation": "distinct", "question": "I want to see the distinct category values available in specimens.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT DISTINCT category FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT category FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "69ac9800bd7a3ee23ccc_hinglish", "language": "hinglish", "backend": "mysql", "operation": "distinct", "question": "Mujhe specimens mein available category ke distinct values dekhne hain.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT DISTINCT category FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT category FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f1f8414c2d47f5396d69_hinglish", "language": "hinglish", "backend": "supabase", "operation": "date_after", "question": "exhibits mein se wo rows chuno jinke created_at mein 2023-03-15 ya uske baad ki date hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE created_at >= '2023-03-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE created_at >= '2023-03-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "3fdbfd3a4d0df901f143_en", "language": "en", "backend": "mysql", "operation": "gte", "question": "Show the data from specimens where amount is 200 or above.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM specimens WHERE amount >= 200;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE amount >= 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bfb3f8261ddb2fc2bf28_en", "language": "en", "backend": "mysql", "operation": "bottom", "question": "Present the 3 minimal price entries from specimens in ascending order.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens ORDER BY price ASC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens ORDER BY price ASC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bfb3f8261ddb2fc2bf28_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "bottom", "question": "print the 3 minimal price rows in specimens from lowest up", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens ORDER BY price ASC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens ORDER BY price ASC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "eab95ae98810f2228ba6_en", "language": "en", "backend": "mysql", "operation": "year", "question": "Show me the specimens rows for 2026 using start_date.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM specimens WHERE YEAR(start_date) = 2026;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE YEAR(start_date) = 2026;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "24661fe309a7c9930145_en", "language": "en", "backend": "mysql", "operation": "month", "question": "Select entries from specimens where recorded_at corresponds to 1.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens WHERE MONTH(recorded_at) = 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE MONTH(recorded_at) = 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d0346c74f1cfe556b2cb_en", "language": "en", "backend": "mysql", "operation": "project_eq", "question": "List the description from inspections where status is Books.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT description FROM inspections WHERE status = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT description FROM inspections WHERE status = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d09477cb849c0af3827d_en", "language": "en", "backend": "mysql", "operation": "ambiguous", "question": "Print the best available records in exhibits.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "d09477cb849c0af3827d_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "ambiguous", "question": "Give me the top records in exhibits.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "78fd6e66c9d726084543_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "count", "question": "how many lines in exhibits?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT COUNT(*) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "aeeb76897a32b9fd2df4_hi", "language": "hi", "backend": "supabase", "operation": "min", "question": "reservations में amount का न्यूनतम परिमाण क्या है?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT MIN(amount) FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT MIN(amount) FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "41ea5651dda516a4ac30_en", "language": "en", "backend": "supabase", "operation": "date_after", "question": "Extract rows from reservations where order_date falls on 2023-12-15 or beyond.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE order_date >= '2023-12-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE order_date >= '2023-12-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e7d754ed36b247db55d3_hinglish", "language": "hinglish", "backend": "mysql", "operation": "not_null", "question": "specimens ka data present karo jahan description null nahi hai.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens WHERE description IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE description IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b9c176c07809f42564a0_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "top", "question": "I need 5 rows from inspections with amount sorted descending.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2162ee1266aed9320937_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "write", "question": "Remove all records from inspections now.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"clarify\",\"question\":\"This connection is read-only. Would you like a SELECT query instead?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "23c0aa3a4596f995ad2a_en", "language": "en", "backend": "mysql", "operation": "count", "question": "What is the volume of data rows in specimens?", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9aed1992a793fbf97601_en", "language": "en", "backend": "mysql", "operation": "min", "question": "What is the baseline price value in exhibits?", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT MIN(price) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT MAX(price) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "5d97abdb52189efbe117_en", "language": "en", "backend": "mysql", "operation": "group_sum", "question": "Please provide the total amount for each status in exhibits.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT status, SUM(amount) FROM exhibits GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT status, SUM(amount) FROM exhibits GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "58eb6ab2fb9c69becdbd_hinglish", "language": "hinglish", "backend": "supabase", "operation": "date_after", "question": "reservations mein se wo rows chuno jinke created_at mein 2022-08-15 ya uske baad ki date hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE created_at >= '2022-08-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE created_at >= '2022-08-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d30264bbc3b08aedbc51_en", "language": "en", "backend": "mysql", "operation": "sum", "question": "Output the total score for the exhibits entity.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT SUM(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT SUM(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5cb3a450bcdc957b7ba5_hi", "language": "hi", "backend": "mysql", "operation": "missing_schema", "question": "कृपया specimens के लिए SQL लिखने से पहले उपलब्ध तालिकाओं का पता लगाएं, क्योंकि मैंने डेटाबेस स्कीमा नहीं दी है।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "55cd623d82474692b1a9_en", "language": "en", "backend": "mysql", "operation": "count", "question": "What is the volume of data rows in specimens?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e2b48c4ec5252934cb20_en", "language": "en", "backend": "mysql", "operation": "count_eq", "question": "Provide the row count for inspections where city is pending.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT COUNT(*) FROM inspections WHERE city = 'pending';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM inspections WHERE city = 'pending';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e2b48c4ec5252934cb20_hinglish", "language": "hinglish", "backend": "mysql", "operation": "count_eq", "question": "Please count the rows in inspections where city is pending.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT COUNT(*) FROM inspections WHERE city = 'pending';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM inspections WHERE city = 'pending';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "eab95ae98810f2228ba6_hinglish", "language": "hinglish", "backend": "mysql", "operation": "year", "question": "specimens mein start_date mein 2026 wale records dikhao.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM specimens WHERE YEAR(start_date) = 2026;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE YEAR(start_date) = 2026;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "24661fe309a7c9930145_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "month", "question": "extract rows from specimens where recorded_at month is 1", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens WHERE MONTH(recorded_at) = 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE MONTH(recorded_at) = 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "16fbb58e3a07a1240f50_en", "language": "en", "backend": "mysql", "operation": "avg", "question": "Return the average score in exhibits.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT AVG(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT AVG(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "16fbb58e3a07a1240f50_hinglish", "language": "hinglish", "backend": "mysql", "operation": "avg", "question": "exhibits me score ka average return karo.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT AVG(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT AVG(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "feb75e02c031a950e223_en", "language": "en", "backend": "mysql", "operation": "group_sum", "question": "Please provide the total cost for each category in inspections.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT category, SUM(cost) FROM inspections GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT category, SUM(cost) FROM inspections GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "175564da35a6c139e5d8_hi", "language": "hi", "backend": "supabase", "operation": "contains", "question": "inspections से वे पंक्तियाँ दिखाएं जिनमें display_name North के समान है, मामला अंतर नजरअंदाज करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE LOWER(display_name) LIKE LOWER('%North%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE LOWER(display_name) LIKE LOWER('%North%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d09477cb849c0af3827d_hinglish", "language": "hinglish", "backend": "mysql", "operation": "ambiguous", "question": "exhibits mein available best records print karo.", "expected": {"action": "clarify", "question": "Best ka matlab kya hai? Kaunsa column aur order?"}, "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "e71814820a36875f0ee5_hinglish", "language": "hinglish", "backend": "mysql", "operation": "not_null", "question": "specimens ka data present karo jahan description null nahi hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM specimens WHERE description IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE description IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "78fd6e66c9d726084543_hinglish", "language": "hinglish", "backend": "mysql", "operation": "count", "question": "Batao exhibits mein kitni entries hain.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT COUNT(*) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a4cdc86496ddd0c1cc90_hi", "language": "hi", "backend": "mysql", "operation": "avg", "question": "specimens में cost का औसत लौटाएँ।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT AVG(cost) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT AVG(cost) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "253e89e4cdbbd5c2d0f9_hinglish", "language": "hinglish", "backend": "mysql", "operation": "group_count", "question": "specimens me pratyek status ke liye rows ki sankhya dhundo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT status, COUNT(*) FROM specimens GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT status, COUNT(*) FROM specimens GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1d7ab92421915fd47ebe_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "year", "question": "Fetch inspections entries where created_at year is 2024.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections WHERE YEAR(created_at) = 2024;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE YEAR(created_at) = 2024;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "cab55a7429427fbc5093_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "distinct", "question": "i need the distinct department values from inspections", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT DISTINCT department FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7506912696ddf94cfae4_en", "language": "en", "backend": "mysql", "operation": "max", "question": "What is the largest number for amount in exhibits?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MAX(amount) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(amount) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7506912696ddf94cfae4_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "max", "question": "what is the upper bound of amount in exhibits?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MAX(amount) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(amount) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5d97abdb52189efbe117_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "group_sum", "question": "I need to see the total amount for each status in exhibits", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT status, SUM(amount) FROM exhibits GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT status, SUM(amount) FROM exhibits GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e8f90532d34c12209a83_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "gte", "question": "output records from inspections where salary is not below 1500", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE salary >= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE salary >= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c0b9cf3b44d96839f256_hinglish", "language": "hinglish", "backend": "mysql", "operation": "min", "question": "inspections mein balance ka minimum reading kya hai?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MIN(balance) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(balance) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b45bf4e81303e2b4da0f_en", "language": "en", "backend": "mysql", "operation": "gt", "question": "Bring up the rows in specimens where price is over 1.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM specimens WHERE price > 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE price > 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8d2022223833a85bd2d1_en", "language": "en", "backend": "mysql", "operation": "max", "question": "What is the largest number for salary in exhibits?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT MAX(salary) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT MAX(salary) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8d2022223833a85bd2d1_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "max", "question": "what is the upper bound of salary in exhibits?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT MAX(salary) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT MAX(salary) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bc8593644a72fe8b9c78_hinglish", "language": "hinglish", "backend": "mysql", "operation": "min", "question": "exhibits mein score ka minimum reading kya hai?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MIN(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "aa867c871299c26d5402_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "lt", "question": "can you select the rows from specimens where rating is less than 1000?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE rating < 1000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE rating < 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "3fdbfd3a4d0df901f143_hinglish", "language": "hinglish", "backend": "mysql", "operation": "gte", "question": "specimens se rows dikhao jahan amount 200 ya usse bada hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM specimens WHERE amount >= 200;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE amount >= 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e114d1f216088c4302b2_en", "language": "en", "backend": "mysql", "operation": "distinct", "question": "I want to see the distinct department values available in specimens.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT DISTINCT department FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT DISTINCT department FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e114d1f216088c4302b2_hinglish", "language": "hinglish", "backend": "mysql", "operation": "distinct", "question": "Mujhe specimens mein available department ke distinct values dekhne hain.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT DISTINCT department FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT DISTINCT department FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4c5f91f7793bd0200f31_hinglish", "language": "hinglish", "backend": "mysql", "operation": "top", "question": "Top 1 rows specimens se lo, amount ke hisaab se highest first.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM specimens ORDER BY amount DESC LIMIT 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY amount DESC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d0346c74f1cfe556b2cb_hinglish", "language": "hinglish", "backend": "mysql", "operation": "project_eq", "question": "inspections se description ki list do jahan status Books hai.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT description FROM inspections WHERE status = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT description FROM inspections WHERE status = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6519cd640e9e4ab22a6e_en", "language": "en", "backend": "mysql", "operation": "gt", "question": "Bring up the rows in specimens where score is over 20.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens WHERE score > 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE score > 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "feb75e02c031a950e223_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "group_sum", "question": "I need to see the total cost for each category in inspections", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT category, SUM(cost) FROM inspections GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT category, SUM(cost) FROM inspections GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "860c064ccde26d474c46_hinglish", "language": "hinglish", "backend": "mysql", "operation": "project", "question": "bas exhibits ka display_name do please", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT display_name FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a6079b58d4365a8f70c8_hinglish", "language": "hinglish", "backend": "mysql", "operation": "null", "question": "inspections se title null wali rows select karo.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "508d2834a4743665db1f_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "distinct", "question": "i need the distinct kind values from exhibits", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT DISTINCT kind FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT DISTINCT kind FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1d7ab92421915fd47ebe_en", "language": "en", "backend": "mysql", "operation": "year", "question": "Show me the inspections rows for 2024 using created_at.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections WHERE YEAR(created_at) = 2024;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE YEAR(created_at) = 2024;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1d7ab92421915fd47ebe_hinglish", "language": "hinglish", "backend": "mysql", "operation": "year", "question": "inspections mein created_at mein 2024 wale records dikhao.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections WHERE YEAR(created_at) = 2024;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE YEAR(created_at) = 2024;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "35406fa024e66bcb5dd7_hi", "language": "hi", "backend": "supabase", "operation": "lt", "question": "specimens से उन पंक्तियों का क्वेरी करें जहाँ score 10000 से कम है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE score < 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE score < 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8efcf517aee64807a8e5_hi", "language": "hi", "backend": "mysql", "operation": "avg", "question": "specimens में balance का औसत लौटाएँ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT AVG(balance) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT AVG(balance) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6972a0b38fd4a8758723_en", "language": "en", "backend": "mysql", "operation": "group_sum", "question": "Please provide the total score for each kind in exhibits.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT kind, SUM(score) FROM exhibits GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT kind, SUM(score) FROM exhibits GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d8f9a887c52e766a8e14_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "year", "question": "Fetch exhibits entries where start_date year is 2023.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM exhibits WHERE YEAR(start_date) = 2023;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE YEAR(start_date) = 2023;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "81b12886ddb859bb71db_en", "language": "en", "backend": "mysql", "operation": "count_eq", "question": "Provide the row count for inspections where city is pending.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM inspections WHERE city = 'pending';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM inspections WHERE city = 'pending';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "81b12886ddb859bb71db_hinglish", "language": "hinglish", "backend": "mysql", "operation": "count_eq", "question": "Please count the rows in inspections where city is pending.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM inspections WHERE city = 'pending';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM inspections WHERE city = 'pending';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7506912696ddf94cfae4_hinglish", "language": "hinglish", "backend": "mysql", "operation": "max", "question": "exhibits mein amount ka sabse bada number kya hai?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MAX(amount) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(amount) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5890e52f983d16adbae7_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "top", "question": "I need 3 rows from inspections with quantity sorted descending.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY quantity DESC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY quantity DESC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2164b376f732a1c371ab_hinglish", "language": "hinglish", "backend": "mysql", "operation": "group_count", "question": "specimens me pratyek kind ke liye rows ki sankhya dhundo.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT kind, COUNT(*) FROM specimens GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM specimens GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5d97abdb52189efbe117_hinglish", "language": "hinglish", "backend": "mysql", "operation": "group_sum", "question": "exhibits me status ke hisaab se amount ka total do please.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT status, SUM(amount) FROM exhibits GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT status, SUM(amount) FROM exhibits GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "15b265bd8bbdb3515605_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "project_eq", "question": "give me display_name from inspections where kind is Electronics", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT display_name FROM inspections WHERE kind = 'Electronics';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT display_name FROM inspections WHERE kind = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d5fc4318c9aedb495460_en", "language": "en", "backend": "mysql", "operation": "lt", "question": "Query inspections for rows where price is less than 50.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE price < 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE price < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0352841d74f7bd7955ea_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "lte", "question": "show the inspections table rows where score is at most 3", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections WHERE score <= 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE score <= 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d61a27de98bb7d44f15d_en", "language": "en", "backend": "mysql", "operation": "count", "question": "What is the volume of data rows in specimens?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM specimens WHERE department = 'volume';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "d30264bbc3b08aedbc51_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sum", "question": "exhibits entity ke liye score ka total output do.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT SUM(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT SUM(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "64372a29664a7eaddbe2_hi", "language": "hi", "backend": "mysql", "operation": "avg", "question": "specimens में amount का औसत लौटाएँ।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT AVG(amount) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT AVG(amount) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c0b9cf3b44d96839f256_en", "language": "en", "backend": "mysql", "operation": "min", "question": "What is the baseline balance value in inspections?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MIN(balance) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(balance) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "cfb6d9bde68e2cdc5bf9_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "project_eq", "question": "give me name from reservations where category is Books", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT name FROM reservations WHERE category = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT name FROM reservations WHERE category = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b45bf4e81303e2b4da0f_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "gt", "question": "show me the entries in specimens where price is strictly greater than 1", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM specimens WHERE price > 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE price > 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "864b7a7bf739071bb030_hi", "language": "hi", "backend": "supabase", "operation": "and", "question": "specimens में वे पंक्तियाँ खोजें जहाँ status Pune है और balance 0 से ऊपर है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE status = 'Pune' AND balance > 0;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE status = 'Pune' AND balance > 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8d2022223833a85bd2d1_hinglish", "language": "hinglish", "backend": "mysql", "operation": "max", "question": "exhibits mein salary ka sabse bada number kya hai?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT MAX(salary) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT MAX(salary) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bc8593644a72fe8b9c78_en", "language": "en", "backend": "mysql", "operation": "min", "question": "What is the baseline score value in exhibits?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MIN(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d96328fdb5748b255271_en", "language": "en", "backend": "mysql", "operation": "group_sum", "question": "Please provide the total salary for each category in inspections.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT category, SUM(salary) FROM inspections GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT category, SUM(salary) FROM inspections GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "24661fe309a7c9930145_hinglish", "language": "hinglish", "backend": "mysql", "operation": "month", "question": "Show me the data from specimens where recorded_at ka month 1 hai.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens WHERE MONTH(recorded_at) = 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE MONTH(recorded_at) = 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "29bdf95ec36d6924a576_hi", "language": "hi", "backend": "supabase", "operation": "contains", "question": "exhibits से वे पंक्तियाँ दिखाएं जिनमें item_name O'Reilly के समान है, मामला अंतर नजरअंदाज करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE LOWER(item_name) LIKE LOWER('%O''Reilly%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE LOWER(item_name) LIKE LOWER('%O''Reilly%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e9cfe7f6c91a2434d315_hi", "language": "hi", "backend": "supabase", "operation": "final", "question": "डेटाबेस टूल ने 102 पंक्तियां वापस कीं। कितनी पंक्तियां लौटीं?", "expected": {"action": "answer", "text": "टूल ने 102 पंक्तियाँ लौटाईं।"}, "output": "{\"action\":\"answer\",\"text\":\"टूल ने 102 पंक्तियाँ लौटाईं।\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "e6380b8531b4b3eb7e38_hi", "language": "hi", "backend": "mysql", "operation": "project", "question": "सिर्फ specimens का name दिखाइए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT name FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT name FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "19d562dac8a4125562f6_en", "language": "en", "backend": "mysql", "operation": "eq", "question": "Bring up all rows from inspections where city has the value Delhi.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM inspections WHERE city = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE city = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "19d562dac8a4125562f6_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "eq", "question": "Get the full list of rows in inspections where city is Delhi.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM inspections WHERE city = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE city = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6519cd640e9e4ab22a6e_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "gt", "question": "show me the entries in specimens where score is strictly greater than 20", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens WHERE score > 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE score > 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "28cb6c4bf0d7caac91c9_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "or", "question": "Display rows from exhibits where kind is Mumbai or Electronics.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'Electronics');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'Electronics');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8439e708b96823b39172_en", "language": "en", "backend": "mysql", "operation": "sort_asc", "question": "View all records in inspections sorted by amount from least to greatest.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "feb75e02c031a950e223_hinglish", "language": "hinglish", "backend": "mysql", "operation": "group_sum", "question": "inspections me category ke hisaab se cost ka total do please.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT category, SUM(cost) FROM inspections GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT category, SUM(cost) FROM inspections GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a76838f987782ef2d2ed_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "describe", "question": "Look up the columns and types for inspections.", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "inspections"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "860c064ccde26d474c46_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "project", "question": "display display_name from exhibits and nothing else", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT display_name FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9eee465dd6b4c3a090c6_en", "language": "en", "backend": "mysql", "operation": "eq", "question": "Bring up all rows from specimens where region has the value Pune.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM specimens WHERE region = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE region = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9eee465dd6b4c3a090c6_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "eq", "question": "Get the full list of rows in specimens where region is Pune.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM specimens WHERE region = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE region = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a6079b58d4365a8f70c8_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "null", "question": "Show me the rows from inspections where title is null.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "41ea5651dda516a4ac30_hinglish", "language": "hinglish", "backend": "supabase", "operation": "date_after", "question": "reservations mein se wo rows chuno jinke order_date mein 2023-12-15 ya uske baad ki date hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE order_date >= '2023-12-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE order_date >= '2023-12-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b9c176c07809f42564a0_hinglish", "language": "hinglish", "backend": "mysql", "operation": "top", "question": "Top 5 rows inspections se lo, amount ke hisaab se highest first.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6972a0b38fd4a8758723_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "group_sum", "question": "I need to see the total score for each kind in exhibits", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT kind, SUM(score) FROM exhibits GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT kind, SUM(score) FROM exhibits GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d8f9a887c52e766a8e14_en", "language": "en", "backend": "mysql", "operation": "year", "question": "Show me the exhibits rows for 2023 using start_date.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM exhibits WHERE YEAR(start_date) = 2023;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE YEAR(start_date) = 2023;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d8f9a887c52e766a8e14_hinglish", "language": "hinglish", "backend": "mysql", "operation": "year", "question": "exhibits mein start_date mein 2023 wale records dikhao.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM exhibits WHERE YEAR(start_date) = 2023;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE YEAR(start_date) = 2023;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2162ee1266aed9320937_en", "language": "en", "backend": "mysql", "operation": "write", "question": "Expunge every row from inspections.", "expected": {"action": "clarify", "question": "This connection is read-only. Would you like a SELECT query instead?"}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE price > 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": false, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false}} +{"id": "2162ee1266aed9320937_hinglish", "language": "hinglish", "backend": "mysql", "operation": "write", "question": "inspections ka pura content hatwa do.", "expected": {"action": "clarify", "question": "Yeh connection read-only hai. Kya SELECT query chahiye?"}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE display_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": false, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": false}} +{"id": "c023a5b6f95173ae856d_en", "language": "en", "backend": "mysql", "operation": "gt", "question": "Bring up the rows in exhibits where score is over 50.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM exhibits WHERE score > 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE score > 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bd760f1ca7aff546e852_en", "language": "en", "backend": "mysql", "operation": "bottom", "question": "Present the 5 minimal balance entries from exhibits in ascending order.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits ORDER BY balance ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits ORDER BY balance ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bd760f1ca7aff546e852_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "bottom", "question": "print the 5 minimal balance rows in exhibits from lowest up", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits ORDER BY balance ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits ORDER BY balance ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "00c24a7cc97ee9debf36_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "contains", "question": "List exhibits rows where title equals pending, ignore case differences.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM exhibits WHERE LOWER(title) LIKE LOWER('%pending%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE LOWER(title) LIKE LOWER('%pending%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "15b265bd8bbdb3515605_en", "language": "en", "backend": "mysql", "operation": "project_eq", "question": "List the display_name from inspections where kind is Electronics.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT display_name FROM inspections WHERE kind = 'Electronics';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT display_name FROM inspections WHERE kind = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e8f90532d34c12209a83_en", "language": "en", "backend": "mysql", "operation": "gte", "question": "Show the data from inspections where salary is 1500 or above.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE salary >= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE salary >= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0352841d74f7bd7955ea_en", "language": "en", "backend": "mysql", "operation": "lte", "question": "Query inspections for rows where score is not greater than 3.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections WHERE score <= 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE score <= 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6f28c044459ce99cf405_en", "language": "en", "backend": "mysql", "operation": "max", "question": "What is the largest number for quantity in exhibits?", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT MAX(quantity) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT MAX(quantity) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6f28c044459ce99cf405_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "max", "question": "what is the upper bound of quantity in exhibits?", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT MAX(quantity) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT MAX(quantity) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "cfb6d9bde68e2cdc5bf9_en", "language": "en", "backend": "mysql", "operation": "project_eq", "question": "List the name from reservations where category is Books.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT name FROM reservations WHERE category = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT name FROM reservations WHERE category = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4e4b83d8af9eb4c5efc8_en", "language": "en", "backend": "mysql", "operation": "bottom", "question": "Present the 5 minimal amount entries from inspections in ascending order.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4e4b83d8af9eb4c5efc8_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "bottom", "question": "print the 5 minimal amount rows in inspections from lowest up", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d96328fdb5748b255271_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "group_sum", "question": "I need to see the total salary for each category in inspections", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT category, SUM(salary) FROM inspections GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT category, SUM(salary) FROM inspections GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2f545db764e4c6cd353d_en", "language": "en", "backend": "mysql", "operation": "gt", "question": "Bring up the rows in exhibits where salary is over 0.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE salary > 0;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE salary > 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "aa867c871299c26d5402_hinglish", "language": "hinglish", "backend": "mysql", "operation": "lt", "question": "specimens se un rows ko query karo jahan rating 1000 se kam hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE rating < 1000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE rating < 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c466f263014e2eb31cf5_en", "language": "en", "backend": "mysql", "operation": "max", "question": "What is the largest number for rating in exhibits?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT MAX(rating) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT MAX(rating) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c466f263014e2eb31cf5_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "max", "question": "what is the upper bound of rating in exhibits?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT MAX(rating) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT MAX(rating) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4c5f91f7793bd0200f31_en", "language": "en", "backend": "mysql", "operation": "top", "question": "Give me the top 1 rows of specimens sorted by amount from high to low.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM specimens ORDER BY amount DESC LIMIT 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY amount DESC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "19d562dac8a4125562f6_hinglish", "language": "hinglish", "backend": "mysql", "operation": "eq", "question": "inspections se wo saari rows lao jahan city ka value Delhi hai.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM inspections WHERE city = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE city = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "28cb6c4bf0d7caac91c9_hinglish", "language": "hinglish", "backend": "mysql", "operation": "or", "question": "exhibits mein kind Mumbai ya Electronics wali rows dikhao na.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'Electronics');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'Electronics');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a7efbaf306821c7ab040_hi", "language": "hi", "backend": "supabase", "operation": "year", "question": "reservations में order_date में 2023 वाले रिकॉर्ड दिखाएं।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE EXTRACT(YEAR FROM order_date) = 2023;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE EXTRACT(YEAR FROM order_date) = 2023;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9f8fa242b083f38bc5f8_en", "language": "en", "backend": "mysql", "operation": "month", "question": "Select entries from inspections where order_date corresponds to 4.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM inspections WHERE MONTH(order_date) = 4;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE MONTH(order_date) = 4;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2c31add7d7b45843dec1_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "project_eq", "question": "give me display_name from inspections where kind is Pune", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT display_name FROM inspections WHERE kind = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM inspections WHERE kind = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b02f421828127e2eb96b_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "lte", "question": "show the exhibits table rows where price is at most 2", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits WHERE price <= 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price <= 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6972a0b38fd4a8758723_hinglish", "language": "hinglish", "backend": "mysql", "operation": "group_sum", "question": "exhibits me kind ke hisaab se score ka total do please.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT kind, SUM(score) FROM exhibits GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT kind, SUM(score) FROM exhibits GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e62cfd2a6789616a503f_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "describe", "question": "Look up the columns and types for reservations.", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "reservations"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "c023a5b6f95173ae856d_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "gt", "question": "show me the entries in exhibits where score is strictly greater than 50", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM exhibits WHERE score > 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE score > 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e8f90532d34c12209a83_hinglish", "language": "hinglish", "backend": "mysql", "operation": "gte", "question": "inspections se rows dikhao jahan salary 1500 ya usse bada hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE salary >= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE salary >= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6f28c044459ce99cf405_hinglish", "language": "hinglish", "backend": "mysql", "operation": "max", "question": "exhibits mein quantity ka sabse bada number kya hai?", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT MAX(quantity) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT MAX(quantity) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5aaaf945b8fba37eadf4_en", "language": "en", "backend": "mysql", "operation": "sort_asc", "question": "View all records in inspections sorted by price from least to greatest.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY price ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY price ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "85e339da2fb28bae5395_hi", "language": "hi", "backend": "supabase", "operation": "date_after", "question": "reservations में से वे पंक्तियाँ चुनें जिनके order_date में 2024-10-15 या उससे बाद की तारीख है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE order_date >= '2024-10-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE order_date >= '2024-10-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "cfb6d9bde68e2cdc5bf9_hinglish", "language": "hinglish", "backend": "mysql", "operation": "project_eq", "question": "reservations se name ki list do jahan category Books hai.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT name FROM reservations WHERE category = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT name FROM reservations WHERE category = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d96328fdb5748b255271_hinglish", "language": "hinglish", "backend": "mysql", "operation": "group_sum", "question": "inspections me category ke hisaab se salary ka total do please.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT category, SUM(salary) FROM inspections GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT category, SUM(salary) FROM inspections GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2ed720de1d1005daa102_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "describe", "question": "Look up the columns and types for reservations.", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "reservations"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "2f545db764e4c6cd353d_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "gt", "question": "show me the entries in exhibits where salary is strictly greater than 0", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE salary > 0;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE salary > 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c466f263014e2eb31cf5_hinglish", "language": "hinglish", "backend": "mysql", "operation": "max", "question": "exhibits mein rating ka sabse bada number kya hai?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT MAX(rating) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT MAX(rating) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "46b2f9dcb5c441ef3e8e_en", "language": "en", "backend": "mysql", "operation": "group_sum", "question": "Please provide the total rating for each kind in exhibits.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT kind, SUM(rating) FROM exhibits GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT kind, SUM(rating) FROM exhibits GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5c6734bdfead38b51752_hi", "language": "hi", "backend": "supabase", "operation": "gte", "question": "specimens से वे पंक्तियाँ दिखाएँ जहाँ quantity 10000 या उससे बड़ा है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE quantity >= 10000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE quantity >= 10000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "28cb6c4bf0d7caac91c9_en", "language": "en", "backend": "mysql", "operation": "or", "question": "Bring up rows from exhibits where kind is either Mumbai or Electronics.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'Electronics');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'Electronics');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8439e708b96823b39172_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sort_asc", "question": "can i see all rows from inspections sorted by amount from small to large", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8439e708b96823b39172_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sort_asc", "question": "inspections se saara data amount ke badhte hue order mein lo aur dikhao.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a76838f987782ef2d2ed_en", "language": "en", "backend": "mysql", "operation": "describe", "question": "Reveal the column structure and types for inspections.", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "inspections"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "9eee465dd6b4c3a090c6_hinglish", "language": "hinglish", "backend": "mysql", "operation": "eq", "question": "specimens se wo saari rows lao jahan region ka value Pune hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM specimens WHERE region = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE region = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7e281ac4ee7d2d835606_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "lte", "question": "show the inspections table rows where rating is at most 1", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE rating <= 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE rating <= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a6079b58d4365a8f70c8_en", "language": "en", "backend": "mysql", "operation": "null", "question": "I want to view the rows in inspections where title is null.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7fb4f6e9f1b68e05cfd9_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sort_desc", "question": "retrieve all records from reservations sorted by salary high to low", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM reservations ORDER BY salary DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "02981b7ccddb9b467260_en", "language": "en", "backend": "mysql", "operation": "sort_asc", "question": "View all records in exhibits sorted by quantity from least to greatest.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits ORDER BY quantity ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits ORDER BY quantity ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9f8fa242b083f38bc5f8_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "month", "question": "extract rows from inspections where order_date month is 4", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM inspections WHERE MONTH(order_date) = 4;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE MONTH(order_date) = 4;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2c31add7d7b45843dec1_en", "language": "en", "backend": "mysql", "operation": "project_eq", "question": "List the display_name from inspections where kind is Pune.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT display_name FROM inspections WHERE kind = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM inspections WHERE kind = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b02f421828127e2eb96b_en", "language": "en", "backend": "mysql", "operation": "lte", "question": "Query exhibits for rows where price is not greater than 2.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits WHERE price <= 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price <= 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "cab55a7429427fbc5093_en", "language": "en", "backend": "mysql", "operation": "distinct", "question": "I want to see the distinct department values available in inspections.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT DISTINCT department FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "cab55a7429427fbc5093_hinglish", "language": "hinglish", "backend": "mysql", "operation": "distinct", "question": "Mujhe inspections mein available department ke distinct values dekhne hain.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT DISTINCT department FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b9c176c07809f42564a0_en", "language": "en", "backend": "mysql", "operation": "top", "question": "Give me the top 5 rows of inspections sorted by amount from high to low.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a060620c638d2dc3759e_hinglish", "language": "hinglish", "backend": "mysql", "operation": "project", "question": "bas reservations ka item_name do please", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT item_name FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5890e52f983d16adbae7_hinglish", "language": "hinglish", "backend": "mysql", "operation": "top", "question": "Top 3 rows inspections se lo, quantity ke hisaab se highest first.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY quantity DESC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY quantity DESC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "35b25e2b46b13ad30caf_en", "language": "en", "backend": "mysql", "operation": "ambiguous", "question": "Print the best available records in reservations.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "35b25e2b46b13ad30caf_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "ambiguous", "question": "Give me the top records in reservations.", "expected": {"action": "clarify", "question": "What does best mean: which column and order?"}, "output": "{\"action\":\"clarify\",\"question\":\"What does best mean: which column and order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "35b25e2b46b13ad30caf_hinglish", "language": "hinglish", "backend": "mysql", "operation": "ambiguous", "question": "reservations mein available best records print karo.", "expected": {"action": "clarify", "question": "Best ka matlab kya hai? Kaunsa column aur order?"}, "output": "{\"action\":\"clarify\",\"question\":\"Best ka matlab kya hai? Kaunsa column aur order?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "15b265bd8bbdb3515605_hinglish", "language": "hinglish", "backend": "mysql", "operation": "project_eq", "question": "inspections se display_name ki list do jahan kind Electronics hai.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT display_name FROM inspections WHERE kind = 'Electronics';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT display_name FROM inspections WHERE kind = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d5fc4318c9aedb495460_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "lt", "question": "can you select the rows from inspections where price is less than 50?", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE price < 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE price < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0352841d74f7bd7955ea_hinglish", "language": "hinglish", "backend": "mysql", "operation": "lte", "question": "inspections se wo rows dhundo jaha score 3 se bada nahi hai.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections WHERE score <= 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE score <= 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2e9e0c695ac50e3982d7_en", "language": "en", "backend": "mysql", "operation": "month", "question": "Select entries from exhibits where order_date corresponds to 12.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits WHERE MONTH(order_date) = 12;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE MONTH(order_date) = 12;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b45bf4e81303e2b4da0f_hinglish", "language": "hinglish", "backend": "mysql", "operation": "gt", "question": "specimens se un rows ko display karo jahan price 1 se more hai.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM specimens WHERE price > 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE price > 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "208e9ccf17a923a46aac_hinglish", "language": "hinglish", "backend": "mysql", "operation": "null", "question": "inspections se item_name null wali rows select karo.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE item_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "46b2f9dcb5c441ef3e8e_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "group_sum", "question": "I need to see the total rating for each kind in exhibits", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT kind, SUM(rating) FROM exhibits GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT kind, SUM(rating) FROM exhibits GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6519cd640e9e4ab22a6e_hinglish", "language": "hinglish", "backend": "mysql", "operation": "gt", "question": "specimens se un rows ko display karo jahan score 20 se more hai.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens WHERE score > 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE score > 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "393d51bc9e30d68b474e_hi", "language": "hi", "backend": "supabase", "operation": "and", "question": "inspections में वे पंक्तियाँ खोजें जहाँ status active है और cost 5 से ऊपर है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE status = 'active' AND cost > 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE status = 'active' AND cost > 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a0bc7565d9d93d546770_hinglish", "language": "hinglish", "backend": "mysql", "operation": "null", "question": "inspections se item_name null wali rows select karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM inspections WHERE item_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "16fbb58e3a07a1240f50_hi", "language": "hi", "backend": "mysql", "operation": "avg", "question": "exhibits में score का औसत लौटाएँ।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT AVG(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT AVG(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b0f56c14239ba5d0227f_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "distinct", "question": "i need the distinct region values from reservations", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT DISTINCT region FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT DISTINCT region FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a76838f987782ef2d2ed_hinglish", "language": "hinglish", "backend": "mysql", "operation": "describe", "question": "Mujhe inspections ke columns aur types ke baare mein info do.", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "inspections"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "5f07a4c7c387316664f1_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "missing_value", "question": "execute filter on exhibits for department, but i have not specified a value", "expected": {"action": "clarify", "question": "Which value should department equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should department equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "860c064ccde26d474c46_en", "language": "en", "backend": "mysql", "operation": "project", "question": "Display display_name from exhibits as the sole result.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT display_name FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7e281ac4ee7d2d835606_en", "language": "en", "backend": "mysql", "operation": "lte", "question": "Query inspections for rows where rating is not greater than 1.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE rating <= 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE rating <= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7e1cd79fe94cad9c7a9e_hi", "language": "hi", "backend": "supabase", "operation": "count_eq", "question": "specimens में region Books वाले रिकॉर्ड्स की संख्या दीजिए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT COUNT(*) FROM specimens WHERE region = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT COUNT(*) FROM specimens WHERE region = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "508d2834a4743665db1f_en", "language": "en", "backend": "mysql", "operation": "distinct", "question": "I want to see the distinct kind values available in exhibits.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT DISTINCT kind FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT DISTINCT kind FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "508d2834a4743665db1f_hinglish", "language": "hinglish", "backend": "mysql", "operation": "distinct", "question": "Mujhe exhibits mein available kind ke distinct values dekhne hain.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT DISTINCT kind FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT DISTINCT kind FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7fb4f6e9f1b68e05cfd9_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sort_desc", "question": "reservations ki poori table salary highest se lowest order mein dikhao.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM reservations ORDER BY salary DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f9fa9654c0e96fd0477b_en", "language": "en", "backend": "mysql", "operation": "eq", "question": "Bring up all rows from inspections where region has the value Books.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE region = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE region = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f9fa9654c0e96fd0477b_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "eq", "question": "Get the full list of rows in inspections where region is Books.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE region = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE region = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8a2e6d77aa1c3a517ecb_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "having", "question": "Tell me the category groups in exhibits with more than 100 rows and their count values.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT category, COUNT(*) FROM exhibits GROUP BY category HAVING COUNT(*) > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM exhibits GROUP BY category HAVING COUNT(*) > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8a2e6d77aa1c3a517ecb_hinglish", "language": "hinglish", "backend": "mysql", "operation": "having", "question": "Please exhibits me 100 se zyada rows wale category groups aur unki count ko display karo.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT category, COUNT(*) FROM exhibits GROUP BY category HAVING COUNT(*) > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM exhibits GROUP BY category HAVING COUNT(*) > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e62cfd2a6789616a503f_en", "language": "en", "backend": "mysql", "operation": "describe", "question": "Reveal the column structure and types for reservations.", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "reservations"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "a060620c638d2dc3759e_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "project", "question": "display item_name from reservations and nothing else", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT item_name FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c023a5b6f95173ae856d_hinglish", "language": "hinglish", "backend": "mysql", "operation": "gt", "question": "exhibits se un rows ko display karo jahan score 50 se more hai.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM exhibits WHERE score > 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE score > 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "affa5e5e14444cd82bb4_en", "language": "en", "backend": "mysql", "operation": "lt", "question": "Query reservations for rows where salary is less than 10.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE salary < 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE salary < 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "45c3d3db5419d8a8dfeb_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "or", "question": "Display rows from exhibits where kind is Mumbai or O'Reilly.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'O''Reilly');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'O''Reilly');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d5fc4318c9aedb495460_hinglish", "language": "hinglish", "backend": "mysql", "operation": "lt", "question": "inspections se un rows ko query karo jahan price 50 se kam hai.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE price < 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE price < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "70da7b441b3a995da19a_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "and", "question": "find rows in exhibits where city is O'Reilly and quantity is above 2", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM exhibits WHERE city = 'O''Reilly' AND quantity > 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE city = 'O''Reilly' AND quantity > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4e9184b509f43a710021_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sort_desc", "question": "retrieve all records from reservations sorted by balance high to low", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations ORDER BY balance DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY balance DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5aaaf945b8fba37eadf4_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sort_asc", "question": "can i see all rows from inspections sorted by price from small to large", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY price ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY price ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5aaaf945b8fba37eadf4_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sort_asc", "question": "inspections se saara data price ke badhte hue order mein lo aur dikhao.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY price ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY price ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a29fdee3339da6711080_hinglish", "language": "hinglish", "backend": "mysql", "operation": "bottom", "question": "rating ke badhte order me specimens ki 20 sabse kam rows, lowest se shuru karke, lao.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens ORDER BY rating ASC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY rating ASC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2e9e0c695ac50e3982d7_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "month", "question": "extract rows from exhibits where order_date month is 12", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits WHERE MONTH(order_date) = 12;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE MONTH(order_date) = 12;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0294353d6bde2cd5185c_en", "language": "en", "backend": "mysql", "operation": "lt", "question": "Query reservations for rows where cost is less than 1.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE cost < 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE cost < 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "208e9ccf17a923a46aac_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "null", "question": "Show me the rows from inspections where item_name is null.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE item_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ba705525122da37a7146_en", "language": "en", "backend": "mysql", "operation": "sort_asc", "question": "View all records in reservations sorted by cost from least to greatest.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM reservations ORDER BY cost ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY cost ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2ed720de1d1005daa102_en", "language": "en", "backend": "mysql", "operation": "describe", "question": "Reveal the column structure and types for reservations.", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "reservations"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "2f545db764e4c6cd353d_hinglish", "language": "hinglish", "backend": "mysql", "operation": "gt", "question": "exhibits se un rows ko display karo jahan salary 0 se more hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE salary > 0;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE salary > 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8643d909ff9369a5491c_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "and", "question": "find rows in inspections where kind is Mumbai and rating is above 1000", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE kind = 'Mumbai' AND rating > 1000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE kind = 'Mumbai' AND rating > 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bfb3f8261ddb2fc2bf28_hinglish", "language": "hinglish", "backend": "mysql", "operation": "bottom", "question": "price ke badhte order me specimens ki 3 sabse kam rows, lowest se shuru karke, lao.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens ORDER BY price ASC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens ORDER BY price ASC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "46b2f9dcb5c441ef3e8e_hinglish", "language": "hinglish", "backend": "mysql", "operation": "group_sum", "question": "exhibits me kind ke hisaab se rating ka total do please.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT kind, SUM(rating) FROM exhibits GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT kind, SUM(rating) FROM exhibits GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a0bc7565d9d93d546770_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "null", "question": "Show me the rows from inspections where item_name is null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM inspections WHERE item_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f7257c4a2eded61f5c86_hi", "language": "hi", "backend": "supabase", "operation": "sort_desc", "question": "specimens की पूरी सारणी score के उच्चतम से निम्नतम क्रम में दिखाएँ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens ORDER BY score DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens ORDER BY score DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b2725cc841b5aaa2fba6_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "and", "question": "find rows in inspections where kind is Electronics and price is above 2", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE kind = 'Electronics' AND price > 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE kind = 'Electronics' AND price > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "78fd6e66c9d726084543_en", "language": "en", "backend": "mysql", "operation": "count", "question": "What is the volume of data rows in exhibits?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT COUNT(*) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7fb4f6e9f1b68e05cfd9_en", "language": "en", "backend": "mysql", "operation": "sort_desc", "question": "Display the entire table reservations sorted by salary from highest to lowest.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM reservations ORDER BY salary DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "02981b7ccddb9b467260_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sort_asc", "question": "can i see all rows from exhibits sorted by quantity from small to large", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits ORDER BY quantity ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits ORDER BY quantity ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "02981b7ccddb9b467260_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sort_asc", "question": "exhibits se saara data quantity ke badhte hue order mein lo aur dikhao.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits ORDER BY quantity ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits ORDER BY quantity ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9f8fa242b083f38bc5f8_hinglish", "language": "hinglish", "backend": "mysql", "operation": "month", "question": "Show me the data from inspections where order_date ka month 4 hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM inspections WHERE MONTH(order_date) = 4;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE MONTH(order_date) = 4;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f9fa9654c0e96fd0477b_hinglish", "language": "hinglish", "backend": "mysql", "operation": "eq", "question": "inspections se wo saari rows lao jahan region ka value Books hai.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE region = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE region = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2c31add7d7b45843dec1_hinglish", "language": "hinglish", "backend": "mysql", "operation": "project_eq", "question": "inspections se display_name ki list do jahan kind Pune hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT display_name FROM inspections WHERE kind = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM inspections WHERE kind = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b02f421828127e2eb96b_hinglish", "language": "hinglish", "backend": "mysql", "operation": "lte", "question": "exhibits se wo rows dhundo jaha price 2 se bada nahi hai.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits WHERE price <= 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price <= 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "475287613764159ee5eb_hi", "language": "hi", "backend": "supabase", "operation": "group_count", "question": "reservations में हर category के लिए पंक्तियों की संख्या खोजें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT category, COUNT(*) FROM reservations GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT category, COUNT(*) FROM reservations GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e62cfd2a6789616a503f_hinglish", "language": "hinglish", "backend": "mysql", "operation": "describe", "question": "Mujhe reservations ke columns aur types ke baare mein info do.", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "reservations"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "1695d78b68e4d8fca6f0_hi", "language": "hi", "backend": "supabase", "operation": "and", "question": "exhibits में वे पंक्तियाँ खोजें जहाँ region Delhi है और amount 1500 से ऊपर है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE region = 'Delhi' AND amount > 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE region = 'Delhi' AND amount > 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "45c3d3db5419d8a8dfeb_hinglish", "language": "hinglish", "backend": "mysql", "operation": "or", "question": "exhibits mein kind Mumbai ya O'Reilly wali rows dikhao na.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'O''Reilly');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'O''Reilly');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "23c0aa3a4596f995ad2a_hi", "language": "hi", "backend": "mysql", "operation": "count", "question": "specimens में कितनी लाइनें मौजूद हैं?", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5890e52f983d16adbae7_en", "language": "en", "backend": "mysql", "operation": "top", "question": "Give me the top 3 rows of inspections sorted by quantity from high to low.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY quantity DESC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY quantity DESC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "70da7b441b3a995da19a_en", "language": "en", "backend": "mysql", "operation": "and", "question": "Show me the exhibits rows where city is O'Reilly and quantity exceeds 2.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM exhibits WHERE city = 'O''Reilly' AND quantity > 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE city = 'O''Reilly' AND quantity > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4e9184b509f43a710021_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sort_desc", "question": "reservations ki poori table balance highest se lowest order mein dikhao.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations ORDER BY balance DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY balance DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "55cd623d82474692b1a9_hi", "language": "hi", "backend": "mysql", "operation": "count", "question": "specimens में कितनी लाइनें मौजूद हैं?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM specimens WHERE city = 'Agra';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false, "error": "no such column: city"}} +{"id": "ba705525122da37a7146_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sort_asc", "question": "reservations se saara data cost ke badhte hue order mein lo aur dikhao.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM reservations ORDER BY cost ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY cost ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2ed720de1d1005daa102_hinglish", "language": "hinglish", "backend": "mysql", "operation": "describe", "question": "Mujhe reservations ke columns aur types ke baare mein info do.", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "reservations"}}, "output": "{\"action\":\"call\",\"name\":\"warehoation.describe_table\",\"arguments\":{\"table\":\"reservations\"}}", "metrics": {"json_valid": true, "schema_valid": false, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false, "error": "Unknown tool"}} +{"id": "8643d909ff9369a5491c_en", "language": "en", "backend": "mysql", "operation": "and", "question": "Show me the inspections rows where kind is Mumbai and rating exceeds 1000.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE kind = 'Mumbai' AND rating > 1000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE kind = 'Mumbai' AND rating > 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "30a7ec8a99d4ee6b6d0e_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "date_after", "question": "Query rows in inspections where order_date is 2026-12-15 onwards.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM inspections WHERE order_date >= '2026-12-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE order_date >= '2026-12-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7e281ac4ee7d2d835606_hinglish", "language": "hinglish", "backend": "mysql", "operation": "lte", "question": "inspections se wo rows dhundo jaha rating 1 se bada nahi hai.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE rating <= 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE rating <= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b2725cc841b5aaa2fba6_en", "language": "en", "backend": "mysql", "operation": "and", "question": "Show me the inspections rows where kind is Electronics and price exceeds 2.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE kind = 'Electronics' AND price > 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE kind = 'Electronics' AND price > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "82c104614df1ce6e2762_hi", "language": "hi", "backend": "supabase", "operation": "gt", "question": "specimens से उन पंक्तियों को प्रदर्शित करें जहाँ salary 100 से ज्यादा है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE salary > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specimens\",\"query\":\"SELECT * FROM specimens WHERE salary > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c01aaa64706cccb8ac54_en", "language": "en", "backend": "mysql", "operation": "between", "question": "Select all items from specimens where the balance column has a value between 0 and 50, inclusive.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens WHERE balance BETWEEN 0 AND 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE balance BETWEEN 0 AND 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6f34909f5b1104bf7ac1_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "or", "question": "Display rows from reservations where region is Delhi or pending.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE region IN ('Delhi', 'pending');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE region IN ('Delhi', 'pending');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6f34909f5b1104bf7ac1_hinglish", "language": "hinglish", "backend": "mysql", "operation": "or", "question": "reservations mein region Delhi ya pending wali rows dikhao na.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE region IN ('Delhi', 'pending');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE region IN ('Delhi', 'pending');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "cd5bb3382b9e6c8604e8_hi", "language": "hi", "backend": "mysql", "operation": "max", "question": "specimens में rating का सबसे बड़ा नंबर क्या है?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MAX(rating) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(rating) FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "45c3d3db5419d8a8dfeb_en", "language": "en", "backend": "mysql", "operation": "or", "question": "Bring up rows from exhibits where kind is either Mumbai or O'Reilly.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'O''Reilly');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'O''Reilly');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "223765dc6e033f0ab2fd_hi", "language": "hi", "backend": "supabase", "operation": "month", "question": "मुझे specimens से वे डेटा पंक्तियां दें जहाँ created_at का महीना 9 है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT * FROM specimens WHERE EXTRACT(MONTH FROM created_at) = 9;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmens\",\"query\":\"SELECT * FROM specimens WHERE EXTRACT(MONTH FROM created_at) = 9;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "00c24a7cc97ee9debf36_hinglish", "language": "hinglish", "backend": "mysql", "operation": "contains", "question": "Output karo exhibits mein title mein pending match hone wali rows, case insensitive raho.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM exhibits WHERE LOWER(title) LIKE LOWER('%pending%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE LOWER(title) LIKE LOWER('%pending%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "49c2e66e48d062e5c69c_hi", "language": "hi", "backend": "mysql", "operation": "write", "question": "specimens का पूरा कंटेंट डिलीट करें।", "expected": {"action": "clarify", "question": "यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "42cda457b51f6f9d3056_en", "language": "en", "backend": "mysql", "operation": "eq", "question": "Bring up all rows from inspections where region has the value O'Reilly.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM inspections WHERE region = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE region = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "42cda457b51f6f9d3056_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "eq", "question": "Get the full list of rows in inspections where region is O'Reilly.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM inspections WHERE region = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE region = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "04b2e5886e750df09a51_hinglish", "language": "hinglish", "backend": "mysql", "operation": "null", "question": "reservations se display_name null wali rows select karo.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations WHERE display_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE display_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4e9184b509f43a710021_en", "language": "en", "backend": "mysql", "operation": "sort_desc", "question": "Display the entire table reservations sorted by balance from highest to lowest.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations ORDER BY balance DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY balance DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2e9e0c695ac50e3982d7_hinglish", "language": "hinglish", "backend": "mysql", "operation": "month", "question": "Show me the data from exhibits where order_date ka month 12 hai.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits WHERE MONTH(order_date) = 12;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE MONTH(order_date) = 12;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9d0222589e9977ea3f57_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "lte", "question": "show the reservations table rows where balance is at most 1500", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM reservations WHERE balance <= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE balance <= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "208e9ccf17a923a46aac_en", "language": "en", "backend": "mysql", "operation": "null", "question": "I want to view the rows in inspections where item_name is null.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE item_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8f9ab277b699f2cbe988_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sort_desc", "question": "retrieve all records from reservations sorted by score high to low", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations ORDER BY score DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY score DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ba705525122da37a7146_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sort_asc", "question": "can i see all rows from reservations sorted by cost from small to large", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM reservations ORDER BY cost ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY cost ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2ca6667f987742cde1e7_hi", "language": "hi", "backend": "mysql", "operation": "write", "question": "specimens का पूरा कंटेंट डिलीट करें।", "expected": {"action": "clarify", "question": "यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "41762bd116b975652e00_en", "language": "en", "backend": "mysql", "operation": "eq", "question": "Bring up all rows from reservations where status has the value South.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE status = 'South';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE status = 'South';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "41762bd116b975652e00_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "eq", "question": "Get the full list of rows in reservations where status is South.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE status = 'South';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE status = 'South';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "41762bd116b975652e00_hinglish", "language": "hinglish", "backend": "mysql", "operation": "eq", "question": "reservations se wo saari rows lao jahan status ka value South hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE status = 'South';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE status = 'South';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d0346c74f1cfe556b2cb_hi", "language": "hi", "backend": "mysql", "operation": "project_eq", "question": "inspections से description की सूची दें जहाँ status Books है।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT description FROM inspections WHERE status = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT description FROM inspections WHERE status = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d58041d2866c015bc923_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "lte", "question": "show the reservations table rows where amount is at most 100", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE amount <= 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE amount <= 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a0bc7565d9d93d546770_en", "language": "en", "backend": "mysql", "operation": "null", "question": "I want to view the rows in inspections where item_name is null.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM inspections WHERE item_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5f07a4c7c387316664f1_en", "language": "en", "backend": "mysql", "operation": "missing_value", "question": "Execute a filter on exhibits based on department, but I have not specified a value.", "expected": {"action": "clarify", "question": "Which value should department equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should department equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "dd1037fa65f5f5a2a06f_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "project_eq", "question": "give me customer_name from reservations where department is North", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT customer_name FROM reservations WHERE department = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT customer_name FROM reservations WHERE department = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e02d95c6aba4ed336a6a_hi", "language": "hi", "backend": "supabase", "operation": "gte", "question": "reservations से वे पंक्तियाँ दिखाएँ जहाँ cost 50 या उससे बड़ा है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE cost >= 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE cost >= 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "dc59f00f50b6c8f7bc16_en", "language": "en", "backend": "mysql", "operation": "between", "question": "Select all items from exhibits where the amount column has a value between 20 and 520, inclusive.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM exhibits WHERE amount BETWEEN 20 AND 520;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE amount BETWEEN 20 AND 520;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9956c110975a38c3d293_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "top", "question": "I need 3 rows from reservations with cost sorted descending.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM reservations ORDER BY cost DESC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY cost DESC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "3fdbd39c598af9b9ea10_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "missing_value", "question": "execute filter on reservations for status, but i have not specified a value", "expected": {"action": "clarify", "question": "Which value should status equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should status equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "a060620c638d2dc3759e_en", "language": "en", "backend": "mysql", "operation": "project", "question": "Display item_name from reservations as the sole result.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT item_name FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "affa5e5e14444cd82bb4_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "lt", "question": "can you select the rows from reservations where salary is less than 10?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE salary < 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE salary < 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "affa5e5e14444cd82bb4_hinglish", "language": "hinglish", "backend": "mysql", "operation": "lt", "question": "reservations se un rows ko query karo jahan salary 10 se kam hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE salary < 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE salary < 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "42cda457b51f6f9d3056_hinglish", "language": "hinglish", "backend": "mysql", "operation": "eq", "question": "inspections se wo saari rows lao jahan region ka value O'Reilly hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM inspections WHERE region = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE region = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d61a27de98bb7d44f15d_hi", "language": "hi", "backend": "mysql", "operation": "count", "question": "specimens में कितनी लाइनें मौजूद हैं?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT COUNT(*) FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM specimens WHERE department = 'tpecimens';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": false, "success": false}} +{"id": "0294353d6bde2cd5185c_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "lt", "question": "can you select the rows from reservations where cost is less than 1?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE cost < 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE cost < 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0294353d6bde2cd5185c_hinglish", "language": "hinglish", "backend": "mysql", "operation": "lt", "question": "reservations se un rows ko query karo jahan cost 1 se kam hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE cost < 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE cost < 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9d0222589e9977ea3f57_en", "language": "en", "backend": "mysql", "operation": "lte", "question": "Query reservations for rows where balance is not greater than 1500.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM reservations WHERE balance <= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE balance <= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8f9ab277b699f2cbe988_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sort_desc", "question": "reservations ki poori table score highest se lowest order mein dikhao.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations ORDER BY score DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY score DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "039280d3483e998ccd1b_hi", "language": "hi", "backend": "supabase", "operation": "group_count", "question": "reservations में हर department के लिए पंक्तियों की संख्या खोजें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT department, COUNT(*) FROM reservations GROUP BY department;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT department, COUNT(*) FROM reservations GROUP BY department;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d58041d2866c015bc923_en", "language": "en", "backend": "mysql", "operation": "lte", "question": "Query reservations for rows where amount is not greater than 100.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE amount <= 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE amount <= 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b0f56c14239ba5d0227f_en", "language": "en", "backend": "mysql", "operation": "distinct", "question": "I want to see the distinct region values available in reservations.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT DISTINCT region FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT DISTINCT region FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b0f56c14239ba5d0227f_hinglish", "language": "hinglish", "backend": "mysql", "operation": "distinct", "question": "Mujhe reservations mein available region ke distinct values dekhne hain.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT DISTINCT region FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT DISTINCT region FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f4fb7fafb95b5c8287f5_hi", "language": "hi", "backend": "supabase", "operation": "group_count", "question": "reservations में हर department के लिए पंक्तियों की संख्या खोजें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT department, COUNT(*) FROM reservations GROUP BY department;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT department, COUNT(*) FROM reservations GROUP BY department;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "55c82b78d80d85926577_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "date_after", "question": "Query rows in exhibits where start_date is 2026-09-15 onwards.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM exhibits WHERE start_date >= '2026-09-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE start_date >= '2026-09-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "dd1037fa65f5f5a2a06f_en", "language": "en", "backend": "mysql", "operation": "project_eq", "question": "List the customer_name from reservations where department is North.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT customer_name FROM reservations WHERE department = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT customer_name FROM reservations WHERE department = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c01aaa64706cccb8ac54_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "between", "question": "pull the rows from specimens where balance is greater than or equal to 0 and less than or equal to 50", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens WHERE balance BETWEEN 0 AND 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE balance BETWEEN 0 AND 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6f34909f5b1104bf7ac1_en", "language": "en", "backend": "mysql", "operation": "or", "question": "Bring up rows from reservations where region is either Delhi or pending.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE region IN ('Delhi', 'pending');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE region IN ('Delhi', 'pending');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e7d754ed36b247db55d3_hi", "language": "hi", "backend": "mysql", "operation": "not_null", "question": "specimens का डेटा दिखाएं जहाँ description null नहीं है।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens WHERE description IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE description IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "804d318fe46a0e664840_en", "language": "en", "backend": "mysql", "operation": "sort_asc", "question": "View all records in reservations sorted by quantity from least to greatest.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations ORDER BY quantity ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY quantity ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7d4234476674a04380e6_hi", "language": "hi", "backend": "supabase", "operation": "gte", "question": "reservations से वे पंक्तियाँ दिखाएँ जहाँ score 1500 या उससे बड़ा है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE score >= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE score >= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c50090341e5d5516a579_en", "language": "en", "backend": "mysql", "operation": "not_null", "question": "Present the data from reservations where customer_name is not null.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM reservations WHERE customer_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE customer_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c50090341e5d5516a579_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "not_null", "question": "Find the rows in reservations where customer_name is not null.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM reservations WHERE customer_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE customer_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c50090341e5d5516a579_hinglish", "language": "hinglish", "backend": "mysql", "operation": "not_null", "question": "reservations ka data present karo jahan customer_name null nahi hai.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM reservations WHERE customer_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE customer_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bd760f1ca7aff546e852_hinglish", "language": "hinglish", "backend": "mysql", "operation": "bottom", "question": "balance ke badhte order me exhibits ki 5 sabse kam rows, lowest se shuru karke, lao.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits ORDER BY balance ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits ORDER BY balance ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "00c24a7cc97ee9debf36_en", "language": "en", "backend": "mysql", "operation": "contains", "question": "List all rows in exhibits where title is equal to pending in a case-insensitive way.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM exhibits WHERE LOWER(title) LIKE LOWER('%pending%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE LOWER(title) LIKE LOWER('%pending%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c07bcd68edd3354fcd83_hi", "language": "hi", "backend": "mysql", "operation": "all", "question": "specimens के सभी रिकॉर्ड प्राप्त करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "70da7b441b3a995da19a_hinglish", "language": "hinglish", "backend": "mysql", "operation": "and", "question": "exhibits mein wo rows dhundo jahan city O'Reilly hai aur quantity 2 se upar hai.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM exhibits WHERE city = 'O''Reilly' AND quantity > 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE city = 'O''Reilly' AND quantity > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "04b2e5886e750df09a51_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "null", "question": "Show me the rows from reservations where display_name is null.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations WHERE display_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE display_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9d0222589e9977ea3f57_hinglish", "language": "hinglish", "backend": "mysql", "operation": "lte", "question": "reservations se wo rows dhundo jaha balance 1500 se bada nahi hai.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM reservations WHERE balance <= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE balance <= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5f0b5d842746b96c126b_en", "language": "en", "backend": "mysql", "operation": "between", "question": "Select all items from exhibits where the price column has a value between 1000 and 1100, inclusive.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM exhibits WHERE price BETWEEN 1000 AND 1100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price BETWEEN 1000 AND 1100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8f9ab277b699f2cbe988_en", "language": "en", "backend": "mysql", "operation": "sort_desc", "question": "Display the entire table reservations sorted by score from highest to lowest.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations ORDER BY score DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY score DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4e4b83d8af9eb4c5efc8_hinglish", "language": "hinglish", "backend": "mysql", "operation": "bottom", "question": "amount ke badhte order me inspections ki 5 sabse kam rows, lowest se shuru karke, lao.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e8cfdfba296bc283222a_hi", "language": "hi", "backend": "supabase", "operation": "month", "question": "मुझे exhibits से वे डेटा पंक्तियां दें जहाँ start_date का महीना 7 है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE EXTRACT(MONTH FROM start_date) = 7;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE EXTRACT(MONTH FROM start_date) = 7;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2cf634192448e68935c1_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "final", "question": "db tool returned 5 rows. tell me the total.", "expected": {"action": "answer", "text": "The tool returned 5 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 5 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "2cf634192448e68935c1_hinglish", "language": "hinglish", "backend": "mysql", "operation": "final", "question": "Kitni rows aayin database tool se? Count hai 5.", "expected": {"action": "answer", "text": "Tool ne 5 rows return ki."}, "output": "{\"action\":\"answer\",\"text\":\"Tool ne 5 rows return ki.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "db7eb47a2bad512e837d_hi", "language": "hi", "backend": "mysql", "operation": "all", "question": "specimens के सभी रिकॉर्ड प्राप्त करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8643d909ff9369a5491c_hinglish", "language": "hinglish", "backend": "mysql", "operation": "and", "question": "inspections mein wo rows dhundo jahan kind Mumbai hai aur rating 1000 se upar hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE kind = 'Mumbai' AND rating > 1000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE kind = 'Mumbai' AND rating > 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b10e3bcce6871b79df41_en", "language": "en", "backend": "mysql", "operation": "sort_asc", "question": "View all records in reservations sorted by price from least to greatest.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations ORDER BY price ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY price ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d58041d2866c015bc923_hinglish", "language": "hinglish", "backend": "mysql", "operation": "lte", "question": "reservations se wo rows dhundo jaha amount 100 se bada nahi hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE amount <= 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE amount <= 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f97582c94f2100eb1007_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "having", "question": "Tell me the department groups in exhibits with more than 500 rows and their count values.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f97582c94f2100eb1007_hinglish", "language": "hinglish", "backend": "mysql", "operation": "having", "question": "Please exhibits me 500 se zyada rows wale department groups aur unki count ko display karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a59c6e6bcec70a1667e5_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "final", "question": "db tool returned 35 rows. tell me the total.", "expected": {"action": "answer", "text": "The tool returned 35 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 35 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "a59c6e6bcec70a1667e5_hinglish", "language": "hinglish", "backend": "mysql", "operation": "final", "question": "Kitni rows aayin database tool se? Count hai 35.", "expected": {"action": "answer", "text": "Tool ne 35 rows return ki."}, "output": "{\"action\":\"answer\",\"text\":\"Tool ne 35 rows return ki.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "860c064ccde26d474c46_hi", "language": "hi", "backend": "mysql", "operation": "project", "question": "सिर्फ exhibits का display_name दिखाइए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT display_name FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "dd1037fa65f5f5a2a06f_hinglish", "language": "hinglish", "backend": "mysql", "operation": "project_eq", "question": "reservations se customer_name ki list do jahan department North hai.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT customer_name FROM reservations WHERE department = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT customer_name FROM reservations WHERE department = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "dc59f00f50b6c8f7bc16_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "between", "question": "pull the rows from exhibits where amount is greater than or equal to 20 and less than or equal to 520", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM exhibits WHERE amount BETWEEN 20 AND 520;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE amount BETWEEN 20 AND 520;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b2725cc841b5aaa2fba6_hinglish", "language": "hinglish", "backend": "mysql", "operation": "and", "question": "inspections mein wo rows dhundo jahan kind Electronics hai aur price 2 se upar hai.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE kind = 'Electronics' AND price > 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE kind = 'Electronics' AND price > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e71814820a36875f0ee5_hi", "language": "hi", "backend": "mysql", "operation": "not_null", "question": "specimens का डेटा दिखाएं जहाँ description null नहीं है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM specimens WHERE description IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE description IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "804d318fe46a0e664840_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sort_asc", "question": "reservations se saara data quantity ke badhte hue order mein lo aur dikhao.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations ORDER BY quantity ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY quantity ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8a2e6d77aa1c3a517ecb_en", "language": "en", "backend": "mysql", "operation": "having", "question": "Give me the category groups in exhibits where the row count is higher than 100, along with the count values.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT category, COUNT(*) FROM exhibits GROUP BY category HAVING COUNT(*) > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM exhibits GROUP BY category HAVING COUNT(*) > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7506912696ddf94cfae4_hi", "language": "hi", "backend": "mysql", "operation": "max", "question": "exhibits में amount का सबसे बड़ा नंबर क्या है?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MAX(amount) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MAX(amount) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e92015bce168f8f9a90a_hi", "language": "hi", "backend": "supabase", "operation": "sort_desc", "question": "exhibits की पूरी सारणी score के उच्चतम से निम्नतम क्रम में दिखाएँ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits ORDER BY score DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits ORDER BY score DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4a79981adf7647bac6b9_hi", "language": "hi", "backend": "supabase", "operation": "gt", "question": "exhibits से उन पंक्तियों को प्रदर्शित करें जहाँ amount 500 से ज्यादा है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE amount > 500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE amount > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d30264bbc3b08aedbc51_hi", "language": "hi", "backend": "mysql", "operation": "sum", "question": "exhibits में score के सभी मानों का योग करें।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT SUM(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT SUM(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "fd0bab1cc08d7298f131_hi", "language": "hi", "backend": "supabase", "operation": "top", "question": "शीर्ष 20 पंक्तियाँ exhibits से प्राप्त करें, salary के आधार पर उच्चतम पहले।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits ORDER BY salary DESC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "cfb6d9bde68e2cdc5bf9_hi", "language": "hi", "backend": "mysql", "operation": "project_eq", "question": "reservations से name की सूची दें जहाँ category Books है।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT name FROM reservations WHERE category = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT name FROM reservations WHERE category = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8d2022223833a85bd2d1_hi", "language": "hi", "backend": "mysql", "operation": "max", "question": "exhibits में salary का सबसे बड़ा नंबर क्या है?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT MAX(salary) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT MAX(salary) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b10e3bcce6871b79df41_hinglish", "language": "hinglish", "backend": "mysql", "operation": "sort_asc", "question": "reservations se saara data price ke badhte hue order mein lo aur dikhao.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations ORDER BY price ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY price ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "30a7ec8a99d4ee6b6d0e_en", "language": "en", "backend": "mysql", "operation": "date_after", "question": "Extract rows from inspections where order_date falls on 2026-12-15 or beyond.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM inspections WHERE order_date >= '2026-12-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE order_date >= '2026-12-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "dc59f00f50b6c8f7bc16_hinglish", "language": "hinglish", "backend": "mysql", "operation": "between", "question": "exhibits se wo saare items select karo jahan amount column ka value 20 aur 520 ke beech hai, inclusive.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM exhibits WHERE amount BETWEEN 20 AND 520;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE amount BETWEEN 20 AND 520;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9956c110975a38c3d293_hinglish", "language": "hinglish", "backend": "mysql", "operation": "top", "question": "Top 3 rows reservations se lo, cost ke hisaab se highest first.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM reservations ORDER BY cost DESC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY cost DESC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c01aaa64706cccb8ac54_hinglish", "language": "hinglish", "backend": "mysql", "operation": "between", "question": "specimens se wo saare items select karo jahan balance column ka value 0 aur 50 ke beech hai, inclusive.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens WHERE balance BETWEEN 0 AND 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE balance BETWEEN 0 AND 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "804d318fe46a0e664840_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sort_asc", "question": "can i see all rows from reservations sorted by quantity from small to large", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations ORDER BY quantity ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY quantity ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d1f372f544fa10f6df1b_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "join", "question": "output reservations.title and reservations_groups.label by joining reservations.group_id to reservations_groups.id", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT reservations.title, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT reservations.title, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "3fdbd39c598af9b9ea10_en", "language": "en", "backend": "mysql", "operation": "missing_value", "question": "Execute a filter on reservations based on status, but I have not specified a value.", "expected": {"action": "clarify", "question": "Which value should status equal?"}, "output": "{\"action\":\"clarify\",\"question\":\"Which value should status equal?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "0d94a075cd7e5759bdfa_en", "language": "en", "backend": "mysql", "operation": "between", "question": "Select all items from exhibits where the rating column has a value between 10 and 110, inclusive.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM exhibits WHERE rating BETWEEN 10 AND 110;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE rating BETWEEN 10 AND 110;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9aed1992a793fbf97601_hi", "language": "hi", "backend": "mysql", "operation": "min", "question": "exhibits में price का न्यूनतम परिमाण क्या है?", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT MIN(price) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT MIN(price) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "15b265bd8bbdb3515605_hi", "language": "hi", "backend": "mysql", "operation": "project_eq", "question": "inspections से display_name की सूची दें जहाँ kind Electronics है।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT display_name FROM inspections WHERE kind = 'Electronics';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT display_name FROM inspections WHERE kind = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "04b2e5886e750df09a51_en", "language": "en", "backend": "mysql", "operation": "null", "question": "I want to view the rows in reservations where display_name is null.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations WHERE display_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE display_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5f0b5d842746b96c126b_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "between", "question": "pull the rows from exhibits where price is greater than or equal to 1000 and less than or equal to 1100", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM exhibits WHERE price BETWEEN 1000 AND 1100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price BETWEEN 1000 AND 1100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b10e3bcce6871b79df41_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "sort_asc", "question": "can i see all rows from reservations sorted by price from small to large", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations ORDER BY price ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY price ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7a33de7fd7159808b04a_hi", "language": "hi", "backend": "supabase", "operation": "month", "question": "मुझे inspections से वे डेटा पंक्तियां दें जहाँ created_at का महीना 11 है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE EXTRACT(MONTH FROM created_at) = 11;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE EXTRACT(MONTH FROM created_at) = 11;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5f07a4c7c387316664f1_hinglish", "language": "hinglish", "backend": "mysql", "operation": "missing_value", "question": "department ke basis pe exhibits pe ek filter execute karo, lekin maine koi value specify nahi ki hai.", "expected": {"action": "clarify", "question": "department ki kaunsi value chahiye?"}, "output": "{\"action\":\"clarify\",\"question\":\"department ki kaunsi value chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "78fd6e66c9d726084543_hi", "language": "hi", "backend": "mysql", "operation": "count", "question": "exhibits में कितनी लाइनें मौजूद हैं?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT COUNT(*) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "adff4eabe2639c0c128c_hi", "language": "hi", "backend": "mysql", "operation": "all", "question": "inspections के सभी रिकॉर्ड प्राप्त करें।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d1f372f544fa10f6df1b_en", "language": "en", "backend": "supabase", "operation": "join", "question": "Extract reservations.title and reservations_groups.label by joining reservations.group_id to reservations_groups.id.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT reservations.title, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT reservations.title, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2162ee1266aed9320937_hi", "language": "hi", "backend": "mysql", "operation": "write", "question": "inspections का पूरा कंटेंट डिलीट करें।", "expected": {"action": "clarify", "question": "यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "5d97abdb52189efbe117_hi", "language": "hi", "backend": "mysql", "operation": "group_sum", "question": "exhibits में status के अनुसार amount का कुल कृपया दें।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT status, SUM(amount) FROM exhibits GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT status, SUM(amount) FROM exhibits GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "916ff7ce3eea9aa64393_hi", "language": "hi", "backend": "supabase", "operation": "missing_value", "question": "kind के आधार पर reservations पर एक फ़िल्टर निष्पादित करें, लेकिन मैंने कोई मान निर्दिष्ट नहीं किया है।", "expected": {"action": "clarify", "question": "kind का कौन सा मान चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"kind का कौन सा मान चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "6f28c044459ce99cf405_hi", "language": "hi", "backend": "mysql", "operation": "max", "question": "exhibits में quantity का सबसे बड़ा नंबर क्या है?", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT MAX(quantity) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT MAX(quantity) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6d9e81a9f52b8d33060a_hi", "language": "hi", "backend": "mysql", "operation": "all", "question": "exhibits के सभी रिकॉर्ड प्राप्त करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5f0b5d842746b96c126b_hinglish", "language": "hinglish", "backend": "mysql", "operation": "between", "question": "exhibits se wo saare items select karo jahan price column ka value 1000 aur 1100 ke beech hai, inclusive.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM exhibits WHERE price BETWEEN 1000 AND 1100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price BETWEEN 1000 AND 1100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2cf634192448e68935c1_en", "language": "en", "backend": "mysql", "operation": "final", "question": "The database tool returned 5 rows. Can you tell me the number?", "expected": {"action": "answer", "text": "The tool returned 5 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 5 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "ccbb8ac5e43114f43047_hi", "language": "hi", "backend": "supabase", "operation": "between", "question": "exhibits से वे सभी आइटम चुनें जहाँ balance कॉलम का मान 50 और 100 के बीच है, समावेशी।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE balance BETWEEN 50 AND 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": false, "success": false, "error": "'query'"}} +{"id": "c466f263014e2eb31cf5_hi", "language": "hi", "backend": "mysql", "operation": "max", "question": "exhibits में rating का सबसे बड़ा नंबर क्या है?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT MAX(rating) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT MAX(rating) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6dd9f03c26298630777f_hi", "language": "hi", "backend": "supabase", "operation": "between", "question": "inspections से वे सभी आइटम चुनें जहाँ score कॉलम का मान 1 और 101 के बीच है, समावेशी।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE score BETWEEN 1 AND 101;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"project_id\":\"demo_inspections\",\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": false, "success": false, "error": "'query'"}} +{"id": "feb75e02c031a950e223_hi", "language": "hi", "backend": "mysql", "operation": "group_sum", "question": "inspections में category के अनुसार cost का कुल कृपया दें।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT category, SUM(cost) FROM inspections GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT category, SUM(cost) FROM inspections GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "55c82b78d80d85926577_en", "language": "en", "backend": "mysql", "operation": "date_after", "question": "Extract rows from exhibits where start_date falls on 2026-09-15 or beyond.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM exhibits WHERE start_date >= '2026-09-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE start_date >= '2026-09-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a59c6e6bcec70a1667e5_en", "language": "en", "backend": "mysql", "operation": "final", "question": "The database tool returned 35 rows. Can you tell me the number?", "expected": {"action": "answer", "text": "The tool returned 35 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 35 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "a6079b58d4365a8f70c8_hi", "language": "hi", "backend": "mysql", "operation": "null", "question": "inspections से title null वाली पंक्तियाँ चुनें।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE title IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE title IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9956c110975a38c3d293_en", "language": "en", "backend": "mysql", "operation": "top", "question": "Give me the top 3 rows of reservations sorted by cost from high to low.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM reservations ORDER BY cost DESC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY cost DESC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2c31add7d7b45843dec1_hi", "language": "hi", "backend": "mysql", "operation": "project_eq", "question": "inspections से display_name की सूची दें जहाँ kind Pune है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT display_name FROM inspections WHERE kind = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT display_name FROM inspections WHERE kind = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a060620c638d2dc3759e_hi", "language": "hi", "backend": "mysql", "operation": "project", "question": "सिर्फ reservations का item_name दिखाइए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT item_name FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT item_name FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0d94a075cd7e5759bdfa_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "between", "question": "pull the rows from exhibits where rating is greater than or equal to 10 and less than or equal to 110", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM exhibits WHERE rating BETWEEN 10 AND 110;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE rating BETWEEN 10 AND 110;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "01f4f9527d44f09ab6b6_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "having", "question": "Tell me the kind groups in reservations with more than 200 rows and their count values.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT kind, COUNT(*) FROM reservations GROUP BY kind HAVING COUNT(*) > 200;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM reservations GROUP BY kind HAVING COUNT(*) > 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "01f4f9527d44f09ab6b6_hinglish", "language": "hinglish", "backend": "mysql", "operation": "having", "question": "Please reservations me 200 se zyada rows wale kind groups aur unki count ko display karo.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT kind, COUNT(*) FROM reservations GROUP BY kind HAVING COUNT(*) > 200;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM reservations GROUP BY kind HAVING COUNT(*) > 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "139a72e10281049d1fed_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "join", "question": "output specimens.name and specimens_groups.label by joining specimens.group_id to specimens_groups.id", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT specimens.name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT specimens.name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c0b9cf3b44d96839f256_hi", "language": "hi", "backend": "mysql", "operation": "min", "question": "inspections में balance का न्यूनतम परिमाण क्या है?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MIN(balance) FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(balance) FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f55358178d003de7ba56_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "final", "question": "db tool returned 133 rows. tell me the total.", "expected": {"action": "answer", "text": "The tool returned 133 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 133 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "f55358178d003de7ba56_hinglish", "language": "hinglish", "backend": "mysql", "operation": "final", "question": "Kitni rows aayin database tool se? Count hai 133.", "expected": {"action": "answer", "text": "Tool ne 133 rows return ki."}, "output": "{\"action\":\"answer\",\"text\":\"Tool ne 133 rows return ki.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "bc8593644a72fe8b9c78_hi", "language": "hi", "backend": "mysql", "operation": "min", "question": "exhibits में score का न्यूनतम परिमाण क्या है?", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT MIN(score) FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT MIN(score) FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6c37ae504c7cb31a6ef3_hi", "language": "hi", "backend": "mysql", "operation": "list_tables", "question": "इस डेटाबेस की तालिकाओं का विवरण दीजिए।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "6972a0b38fd4a8758723_hi", "language": "hi", "backend": "mysql", "operation": "group_sum", "question": "exhibits में kind के अनुसार score का कुल कृपया दें।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT kind, SUM(score) FROM exhibits GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT kind, SUM(score) FROM exhibits GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0d94a075cd7e5759bdfa_hinglish", "language": "hinglish", "backend": "mysql", "operation": "between", "question": "exhibits se wo saare items select karo jahan rating column ka value 10 aur 110 ke beech hai, inclusive.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM exhibits WHERE rating BETWEEN 10 AND 110;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE rating BETWEEN 10 AND 110;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "139a72e10281049d1fed_en", "language": "en", "backend": "mysql", "operation": "join", "question": "Extract specimens.name and specimens_groups.label by joining specimens.group_id to specimens_groups.id.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT specimens.name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT specimens.name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "40ffef08b12e4a3276e3_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "final", "question": "db tool returned 116 rows. tell me the total.", "expected": {"action": "answer", "text": "The tool returned 116 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 116 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "40ffef08b12e4a3276e3_hinglish", "language": "hinglish", "backend": "mysql", "operation": "final", "question": "Kitni rows aayin database tool se? Count hai 116.", "expected": {"action": "answer", "text": "Tool ne 116 rows return ki."}, "output": "{\"action\":\"answer\",\"text\":\"Tool ne 116 rows return ki.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "b7703ebecadd9433c5f8_hi", "language": "hi", "backend": "supabase", "operation": "top", "question": "शीर्ष 3 पंक्तियाँ reservations से प्राप्त करें, quantity के आधार पर उच्चतम पहले।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations ORDER BY quantity DESC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations ORDER BY quantity DESC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d96328fdb5748b255271_hi", "language": "hi", "backend": "mysql", "operation": "group_sum", "question": "inspections में category के अनुसार salary का कुल कृपया दें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT category, SUM(salary) FROM inspections GROUP BY category;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT category, SUM(salary) FROM inspections GROUP BY category;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e02df083b1ff1b7fbcf8_hi", "language": "hi", "backend": "supabase", "operation": "join_filter", "question": "कृपया exhibits.customer_name दिखाएँ जहाँ exhibits exhibits.group_id के माध्यम से exhibits_groups से जुड़ा है और exhibits_groups.label North है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT exhibits.customer_name FROM exhibits JOIN exhibits_groups ON exhibits.group_id = exhibits_groups.id WHERE exhibits_groups.label = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT exhibits.customer_name FROM exhibits JOIN exhibits_groups ON exhibits.group_id = exhibits_groups.id WHERE exhibits_groups.label = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "30a7ec8a99d4ee6b6d0e_hinglish", "language": "hinglish", "backend": "mysql", "operation": "date_after", "question": "inspections mein se wo rows chuno jinke order_date mein 2026-12-15 ya uske baad ki date hai.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM inspections WHERE order_date >= '2026-12-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE order_date >= '2026-12-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "eab95ae98810f2228ba6_hi", "language": "hi", "backend": "mysql", "operation": "year", "question": "specimens में start_date में 2026 वाले रिकॉर्ड दिखाएं।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM specimens WHERE YEAR(start_date) = 2026;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE YEAR(start_date) = 2026;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9bb888f912ca90f30c9a_hi", "language": "hi", "backend": "supabase", "operation": "top", "question": "शीर्ष 20 पंक्तियाँ reservations से प्राप्त करें, rating के आधार पर उच्चतम पहले।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations ORDER BY rating DESC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations ORDER BY rating DESC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f97582c94f2100eb1007_en", "language": "en", "backend": "mysql", "operation": "having", "question": "Give me the department groups in exhibits where the row count is higher than 500, along with the count values.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "785d62b3d5e4932f5c33_hi", "language": "hi", "backend": "supabase", "operation": "bottom", "question": "rating के बढ़ते क्रम में inspections की 20 सबसे कम पंक्तियां, सबसे कम से शुरू होकर, लें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections ORDER BY rating ASC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections ORDER BY rating ASC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d1f372f544fa10f6df1b_hinglish", "language": "hinglish", "backend": "supabase", "operation": "join", "question": "Kripya reservations.group_id ko reservations_groups.id se join karke reservations.title aur reservations_groups.label retrieve karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT reservations.title, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT reservations.title, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c4210296305105ab7472_hi", "language": "hi", "backend": "mysql", "operation": "ambiguous", "question": "specimens में उपलब्ध सर्वोत्तम रिकॉर्ड प्रिंट करें।", "expected": {"action": "clarify", "question": "सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?"}, "output": "{\"action\":\"clarify\",\"question\":\"सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "3fdbd39c598af9b9ea10_hinglish", "language": "hinglish", "backend": "mysql", "operation": "missing_value", "question": "status ke basis pe reservations pe ek filter execute karo, lekin maine koi value specify nahi ki hai.", "expected": {"action": "clarify", "question": "status ki kaunsi value chahiye?"}, "output": "{\"action\":\"clarify\",\"question\":\"status ki kaunsi value chahiye?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "8a3194b9f492b50b8100_hi", "language": "hi", "backend": "mysql", "operation": "distinct", "question": "मुझे specimens में उपलब्ध category के विशिष्ट मान देखने हैं।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT DISTINCT category FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT category FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "3b3a8e441f0ae5cbaad4_hi", "language": "hi", "backend": "mysql", "operation": "ambiguous", "question": "specimens में उपलब्ध सर्वोत्तम रिकॉर्ड प्रिंट करें।", "expected": {"action": "clarify", "question": "सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?"}, "output": "{\"action\":\"clarify\",\"question\":\"सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "69ac9800bd7a3ee23ccc_hi", "language": "hi", "backend": "mysql", "operation": "distinct", "question": "मुझे specimens में उपलब्ध category के विशिष्ट मान देखने हैं।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT DISTINCT category FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT DISTINCT category FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9bf8948becf74e81a103_hi", "language": "hi", "backend": "mysql", "operation": "list_tables", "question": "इस डेटाबेस की तालिकाओं का विवरण दीजिए।", "expected": {"action": "call", "name": "warehouse.list_tables", "arguments": {"schemas": ["public"]}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.list_tables\",\"arguments\":{\"schemas\":[\"public\"]}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "844b55044ac74adf4493_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "join", "question": "output reservations.item_name and reservations_groups.label by joining reservations.group_id to reservations_groups.id", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "dddd09f34c0e849282a8_hi", "language": "hi", "backend": "supabase", "operation": "gt", "question": "reservations से उन पंक्तियों को प्रदर्शित करें जहाँ salary 20 से ज्यादा है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE salary > 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE salary > 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1d7ab92421915fd47ebe_hi", "language": "hi", "backend": "mysql", "operation": "year", "question": "inspections में created_at में 2024 वाले रिकॉर्ड दिखाएं।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections WHERE YEAR(created_at) = 2024;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE YEAR(created_at) = 2024;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9ebca064e63b0347236b_hi", "language": "hi", "backend": "supabase", "operation": "having", "question": "कृपया specimens में 5 से अधिक पंक्तियों वाले region समूहों और उनकी गणना को प्रदर्शित करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmens\",\"query\":\"SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "f55358178d003de7ba56_en", "language": "en", "backend": "mysql", "operation": "final", "question": "The database tool returned 133 rows. Can you tell me the number?", "expected": {"action": "answer", "text": "The tool returned 133 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 133 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "208e9ccf17a923a46aac_hi", "language": "hi", "backend": "mysql", "operation": "null", "question": "inspections से item_name null वाली पंक्तियाँ चुनें।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE item_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "46b2f9dcb5c441ef3e8e_hi", "language": "hi", "backend": "mysql", "operation": "group_sum", "question": "exhibits में kind के अनुसार rating का कुल कृपया दें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT kind, SUM(rating) FROM exhibits GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT kind, SUM(rating) FROM exhibits GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bd5fbea0dbbb3414c53c_hi", "language": "hi", "backend": "supabase", "operation": "having", "question": "कृपया specimens में 0 से अधिक पंक्तियों वाले region समूहों और उनकी गणना को प्रदर्शित करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_specimens", "query": "SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 0;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_specmemens\",\"query\":\"SELECT region, COUNT(*) FROM specimens GROUP BY region HAVING COUNT(*) > 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": true, "success": false}} +{"id": "a0bc7565d9d93d546770_hi", "language": "hi", "backend": "mysql", "operation": "null", "question": "inspections से item_name null वाली पंक्तियाँ चुनें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM inspections WHERE item_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE item_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "55c82b78d80d85926577_hinglish", "language": "hinglish", "backend": "mysql", "operation": "date_after", "question": "exhibits mein se wo rows chuno jinke start_date mein 2026-09-15 ya uske baad ki date hai.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM exhibits WHERE start_date >= '2026-09-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE start_date >= '2026-09-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "844b55044ac74adf4493_en", "language": "en", "backend": "supabase", "operation": "join", "question": "Extract reservations.item_name and reservations_groups.label by joining reservations.group_id to reservations_groups.id.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "dd1037fa65f5f5a2a06f_hi", "language": "hi", "backend": "mysql", "operation": "project_eq", "question": "reservations से customer_name की सूची दें जहाँ department North है।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT customer_name FROM reservations WHERE department = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT customer_name FROM reservations WHERE department = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d8f9a887c52e766a8e14_hi", "language": "hi", "backend": "mysql", "operation": "year", "question": "exhibits में start_date में 2023 वाले रिकॉर्ड दिखाएं।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM exhibits WHERE YEAR(start_date) = 2023;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE YEAR(start_date) = 2023;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "139a72e10281049d1fed_hinglish", "language": "hinglish", "backend": "mysql", "operation": "join", "question": "Kripya specimens.group_id ko specimens_groups.id se join karke specimens.name aur specimens_groups.label retrieve karo.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT specimens.name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT specimens.name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "40ffef08b12e4a3276e3_en", "language": "en", "backend": "mysql", "operation": "final", "question": "The database tool returned 116 rows. Can you tell me the number?", "expected": {"action": "answer", "text": "The tool returned 116 rows."}, "output": "{\"action\":\"answer\",\"text\":\"The tool returned 116 rows.\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "e114d1f216088c4302b2_hi", "language": "hi", "backend": "mysql", "operation": "distinct", "question": "मुझे specimens में उपलब्ध department के विशिष्ट मान देखने हैं।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT DISTINCT department FROM specimens;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT DISTINCT department FROM specimens;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "28cb6c4bf0d7caac91c9_hi", "language": "hi", "backend": "mysql", "operation": "or", "question": "exhibits में kind Mumbai या Electronics वाली पंक्तियाँ दिखाइए।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'Electronics');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'Electronics');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a76838f987782ef2d2ed_hi", "language": "hi", "backend": "mysql", "operation": "describe", "question": "टेबल inspections के कॉलम संरचना और टाइप्स दिखाइए।", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "inspections"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"inspections\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "01f4f9527d44f09ab6b6_en", "language": "en", "backend": "mysql", "operation": "having", "question": "Give me the kind groups in reservations where the row count is higher than 200, along with the count values.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT kind, COUNT(*) FROM reservations GROUP BY kind HAVING COUNT(*) > 200;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM reservations GROUP BY kind HAVING COUNT(*) > 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d09477cb849c0af3827d_hi", "language": "hi", "backend": "mysql", "operation": "ambiguous", "question": "exhibits में उपलब्ध सर्वोत्तम रिकॉर्ड प्रिंट करें।", "expected": {"action": "clarify", "question": "सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?"}, "output": "{\"action\":\"clarify\",\"question\":\"सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "253e89e4cdbbd5c2d0f9_hi", "language": "hi", "backend": "mysql", "operation": "group_count", "question": "specimens में हर status के लिए पंक्तियों की संख्या खोजें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT status, COUNT(*) FROM specimens GROUP BY status;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT status, COUNT(*) FROM specimens GROUP BY status;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a30eeaf286c79c95a1cd_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "join_filter", "question": "show me inspections.item_name for inspections entries where inspections.group_id links to inspections_groups.id and inspections_groups.label is O'Reilly", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT inspections.item_name FROM inspections JOIN inspections_groups ON inspections.group_id = inspections_groups.id WHERE inspections_groups.label = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT inspections.item_name FROM inspections JOIN inspections_groups ON inspections.group_id = inspections_groups.id WHERE inspections_groups.label = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a30eeaf286c79c95a1cd_hinglish", "language": "hinglish", "backend": "supabase", "operation": "join_filter", "question": "Please inspections.item_name dikhao jahan inspections inspections.group_id ke through inspections_groups se juda hai aur inspections_groups.label O'Reilly hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT inspections.item_name FROM inspections JOIN inspections_groups ON inspections.group_id = inspections_groups.id WHERE inspections_groups.label = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT inspections.item_name FROM inspections JOIN inspections_groups ON inspections.group_id = inspections_groups.id WHERE inspections_groups.label = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e62cfd2a6789616a503f_hi", "language": "hi", "backend": "mysql", "operation": "describe", "question": "टेबल reservations के कॉलम संरचना और टाइप्स दिखाइए।", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "reservations"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "c50090341e5d5516a579_hi", "language": "hi", "backend": "mysql", "operation": "not_null", "question": "reservations का डेटा दिखाएं जहाँ customer_name null नहीं है।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM reservations WHERE customer_name IS NOT NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE customer_name IS NOT NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "04b2e5886e750df09a51_hi", "language": "hi", "backend": "mysql", "operation": "null", "question": "reservations से display_name null वाली पंक्तियाँ चुनें।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations WHERE display_name IS NULL;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE display_name IS NULL;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "307d5b7bd93949f7381a_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "join", "question": "output specimens.customer_name and specimens_groups.label by joining specimens.group_id to specimens_groups.id", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2ed720de1d1005daa102_hi", "language": "hi", "backend": "mysql", "operation": "describe", "question": "टेबल reservations के कॉलम संरचना और टाइप्स दिखाइए।", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "reservations"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"reservations\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "aa867c871299c26d5402_hi", "language": "hi", "backend": "mysql", "operation": "lt", "question": "specimens से उन पंक्तियों का क्वेरी करें जहाँ rating 1000 से कम है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens WHERE rating < 1000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE rating < 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "3fdbfd3a4d0df901f143_hi", "language": "hi", "backend": "mysql", "operation": "gte", "question": "specimens से वे पंक्तियाँ दिखाएँ जहाँ amount 200 या उससे बड़ा है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM specimens WHERE amount >= 200;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE amount >= 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "53220f60789b4f5e4046_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "join", "question": "output specimens.customer_name and specimens_groups.label by joining specimens.group_id to specimens_groups.id", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "19d562dac8a4125562f6_hi", "language": "hi", "backend": "mysql", "operation": "eq", "question": "inspections से वे सभी पंक्तियाँ लाएँ जहाँ city का मान Delhi है।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM inspections WHERE city = 'Delhi';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE city = 'Delhi';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "844b55044ac74adf4493_hinglish", "language": "hinglish", "backend": "supabase", "operation": "join", "question": "Kripya reservations.group_id ko reservations_groups.id se join karke reservations.item_name aur reservations_groups.label retrieve karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a30eeaf286c79c95a1cd_en", "language": "en", "backend": "supabase", "operation": "join_filter", "question": "Give me inspections.item_name for inspections entries where inspections.group_id points to inspections_groups.id and inspections_groups.label is O'Reilly.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT inspections.item_name FROM inspections JOIN inspections_groups ON inspections.group_id = inspections_groups.id WHERE inspections_groups.label = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT inspections.item_name FROM inspections JOIN inspections_groups ON inspections.group_id = inspections_groups.id WHERE inspections_groups.label = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "30dd94e63a065d87109e_hi", "language": "hi", "backend": "mysql", "operation": "sql_error", "question": "पिछला क्वेरी विफल रहा क्योंकि एक कॉलम अज्ञात था। दोबारा प्रयास करने से पहले specimens स्कीमा की पुष्टि करें।", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "specimens"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"saarmens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "2164b376f732a1c371ab_hi", "language": "hi", "backend": "mysql", "operation": "group_count", "question": "specimens में हर kind के लिए पंक्तियों की संख्या खोजें।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT kind, COUNT(*) FROM specimens GROUP BY kind;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM specimens GROUP BY kind;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0352841d74f7bd7955ea_hi", "language": "hi", "backend": "mysql", "operation": "lte", "question": "inspections से वे पंक्तियां खोजें जहां score 3 से बड़ा नहीं है।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections WHERE score <= 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE score <= 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "307d5b7bd93949f7381a_en", "language": "en", "backend": "mysql", "operation": "join", "question": "Extract specimens.customer_name and specimens_groups.label by joining specimens.group_id to specimens_groups.id.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "375b86b5cd0d00cd7868_hi", "language": "hi", "backend": "supabase", "operation": "having", "question": "कृपया exhibits में 2 से अधिक पंक्तियों वाले department समूहों और उनकी गणना को प्रदर्शित करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "53220f60789b4f5e4046_en", "language": "en", "backend": "mysql", "operation": "join", "question": "Extract specimens.customer_name and specimens_groups.label by joining specimens.group_id to specimens_groups.id.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6f162db917ded36d0df0_hi", "language": "hi", "backend": "mysql", "operation": "sql_error", "question": "पिछला क्वेरी विफल रहा क्योंकि एक कॉलम अज्ञात था। दोबारा प्रयास करने से पहले specimens स्कीमा की पुष्टि करें।", "expected": {"action": "call", "name": "warehouse.describe_table", "arguments": {"table": "specimens"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"specmens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": false, "sql_equivalent": null, "success": false}} +{"id": "1523073b9b419fc60ded_hi", "language": "hi", "backend": "mysql", "operation": "count_eq", "question": "specimens में city Books वाले रिकॉर्ड्स की संख्या दीजिए।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT COUNT(*) FROM specimens WHERE city = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM specimens WHERE city = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9eee465dd6b4c3a090c6_hi", "language": "hi", "backend": "mysql", "operation": "eq", "question": "specimens से वे सभी पंक्तियाँ लाएँ जहाँ region का मान Pune है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM specimens WHERE region = 'Pune';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE region = 'Pune';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "cab55a7429427fbc5093_hi", "language": "hi", "backend": "mysql", "operation": "distinct", "question": "मुझे inspections में उपलब्ध department के विशिष्ट मान देखने हैं।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT DISTINCT department FROM inspections;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT DISTINCT department FROM inspections;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "45c3d3db5419d8a8dfeb_hi", "language": "hi", "backend": "mysql", "operation": "or", "question": "exhibits में kind Mumbai या O'Reilly वाली पंक्तियाँ दिखाइए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'O''Reilly');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE kind IN ('Mumbai', 'O''Reilly');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e2b48c4ec5252934cb20_hi", "language": "hi", "backend": "mysql", "operation": "count_eq", "question": "inspections में city pending वाले रिकॉर्ड्स की संख्या दीजिए।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT COUNT(*) FROM inspections WHERE city = 'pending';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT COUNT(*) FROM inspections WHERE city = 'pending';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "508d2834a4743665db1f_hi", "language": "hi", "backend": "mysql", "operation": "distinct", "question": "मुझे exhibits में उपलब्ध kind के विशिष्ट मान देखने हैं।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT DISTINCT kind FROM exhibits;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT DISTINCT kind FROM exhibits;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b02f421828127e2eb96b_hi", "language": "hi", "backend": "mysql", "operation": "lte", "question": "exhibits से वे पंक्तियां खोजें जहां price 2 से बड़ा नहीं है।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits WHERE price <= 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE price <= 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6f34909f5b1104bf7ac1_hi", "language": "hi", "backend": "mysql", "operation": "or", "question": "reservations में region Delhi या pending वाली पंक्तियाँ दिखाइए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE region IN ('Delhi', 'pending');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE region IN ('Delhi', 'pending');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d5fc4318c9aedb495460_hi", "language": "hi", "backend": "mysql", "operation": "lt", "question": "inspections से उन पंक्तियों का क्वेरी करें जहाँ price 50 से कम है।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE price < 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE price < 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "e8f90532d34c12209a83_hi", "language": "hi", "backend": "mysql", "operation": "gte", "question": "inspections से वे पंक्तियाँ दिखाएँ जहाँ salary 1500 या उससे बड़ा है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE salary >= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE salary >= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8439e708b96823b39172_hi", "language": "hi", "backend": "mysql", "operation": "sort_asc", "question": "inspections से सभी डेटा amount के बढ़ते हुए क्रम में लें और दिखाएं।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7e281ac4ee7d2d835606_hi", "language": "hi", "backend": "mysql", "operation": "lte", "question": "inspections से वे पंक्तियां खोजें जहां rating 1 से बड़ा नहीं है।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE rating <= 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE rating <= 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f9fa9654c0e96fd0477b_hi", "language": "hi", "backend": "mysql", "operation": "eq", "question": "inspections से वे सभी पंक्तियाँ लाएँ जहाँ region का मान Books है।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE region = 'Books';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE region = 'Books';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "81b12886ddb859bb71db_hi", "language": "hi", "backend": "mysql", "operation": "count_eq", "question": "inspections में city pending वाले रिकॉर्ड्स की संख्या दीजिए।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT COUNT(*) FROM inspections WHERE city = 'pending';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT COUNT(*) FROM inspections WHERE city = 'pending';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "35b25e2b46b13ad30caf_hi", "language": "hi", "backend": "mysql", "operation": "ambiguous", "question": "reservations में उपलब्ध सर्वोत्तम रिकॉर्ड प्रिंट करें।", "expected": {"action": "clarify", "question": "सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?"}, "output": "{\"action\":\"clarify\",\"question\":\"सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "307d5b7bd93949f7381a_hinglish", "language": "hinglish", "backend": "mysql", "operation": "join", "question": "Kripya specimens.group_id ko specimens_groups.id se join karke specimens.customer_name aur specimens_groups.label retrieve karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "24661fe309a7c9930145_hi", "language": "hi", "backend": "mysql", "operation": "month", "question": "मुझे specimens से वे डेटा पंक्तियां दें जहाँ recorded_at का महीना 1 है।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens WHERE MONTH(recorded_at) = 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE MONTH(recorded_at) = 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "53220f60789b4f5e4046_hinglish", "language": "hinglish", "backend": "mysql", "operation": "join", "question": "Kripya specimens.group_id ko specimens_groups.id se join karke specimens.customer_name aur specimens_groups.label retrieve karo.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "63fa94106f0cf26f7da2_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "join_filter", "question": "show me specimens.title for specimens entries where specimens.group_id links to specimens_groups.id and specimens_groups.label is North", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT specimens.title FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id WHERE specimens_groups.label = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT specimens.title FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id WHERE specimens_groups.label = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "63fa94106f0cf26f7da2_hinglish", "language": "hinglish", "backend": "mysql", "operation": "join_filter", "question": "Please specimens.title dikhao jahan specimens specimens.group_id ke through specimens_groups se juda hai aur specimens_groups.label North hai.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT specimens.title FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id WHERE specimens_groups.label = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT specimens.title FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id WHERE specimens_groups.label = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1d82ef2b9527e797c466_hi", "language": "hi", "backend": "supabase", "operation": "date_after", "question": "inspections में से वे पंक्तियाँ चुनें जिनके order_date में 2022-04-15 या उससे बाद की तारीख है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT * FROM inspections WHERE order_date >= '2022-04-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT * FROM inspections WHERE order_date >= '2022-04-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "70da7b441b3a995da19a_hi", "language": "hi", "backend": "mysql", "operation": "and", "question": "exhibits में वे पंक्तियाँ खोजें जहाँ city O'Reilly है और quantity 2 से ऊपर है।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM exhibits WHERE city = 'O''Reilly' AND quantity > 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE city = 'O''Reilly' AND quantity > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5aaaf945b8fba37eadf4_hi", "language": "hi", "backend": "mysql", "operation": "sort_asc", "question": "inspections से सभी डेटा price के बढ़ते हुए क्रम में लें और दिखाएं।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY price ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY price ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f1f8414c2d47f5396d69_hi", "language": "hi", "backend": "supabase", "operation": "date_after", "question": "exhibits में से वे पंक्तियाँ चुनें जिनके created_at में 2023-03-15 या उससे बाद की तारीख है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_exhibits", "query": "SELECT * FROM exhibits WHERE created_at >= '2023-03-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_exhibits\",\"query\":\"SELECT * FROM exhibits WHERE created_at >= '2023-03-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "41762bd116b975652e00_hi", "language": "hi", "backend": "mysql", "operation": "eq", "question": "reservations से वे सभी पंक्तियाँ लाएँ जहाँ status का मान South है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE status = 'South';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE status = 'South';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8643d909ff9369a5491c_hi", "language": "hi", "backend": "mysql", "operation": "and", "question": "inspections में वे पंक्तियाँ खोजें जहाँ kind Mumbai है और rating 1000 से ऊपर है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM inspections WHERE kind = 'Mumbai' AND rating > 1000;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM inspections WHERE kind = 'Mumbai' AND rating > 1000;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4c5f91f7793bd0200f31_hi", "language": "hi", "backend": "mysql", "operation": "top", "question": "शीर्ष 1 पंक्तियाँ specimens से प्राप्त करें, amount के आधार पर उच्चतम पहले।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM specimens ORDER BY amount DESC LIMIT 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY amount DESC LIMIT 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "63fa94106f0cf26f7da2_en", "language": "en", "backend": "mysql", "operation": "join_filter", "question": "Give me specimens.title for specimens entries where specimens.group_id points to specimens_groups.id and specimens_groups.label is North.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT specimens.title FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id WHERE specimens_groups.label = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT specimens.title FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id WHERE specimens_groups.label = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b2725cc841b5aaa2fba6_hi", "language": "hi", "backend": "mysql", "operation": "and", "question": "inspections में वे पंक्तियाँ खोजें जहाँ kind Electronics है और price 2 से ऊपर है।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM inspections WHERE kind = 'Electronics' AND price > 2;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE kind = 'Electronics' AND price > 2;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "02981b7ccddb9b467260_hi", "language": "hi", "backend": "mysql", "operation": "sort_asc", "question": "exhibits से सभी डेटा quantity के बढ़ते हुए क्रम में लें और दिखाएं।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits ORDER BY quantity ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits ORDER BY quantity ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "797b9b914530f2eb502a_hi", "language": "hi", "backend": "supabase", "operation": "final", "question": "डेटाबेस टूल ने 48 पंक्तियां वापस कीं। कितनी पंक्तियां लौटीं?", "expected": {"action": "answer", "text": "टूल ने 48 पंक्तियाँ लौटाईं।"}, "output": "{\"action\":\"answer\",\"text\":\"टूल ने 48 पंक्तियाँ लौटाईं।\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "affa5e5e14444cd82bb4_hi", "language": "hi", "backend": "mysql", "operation": "lt", "question": "reservations से उन पंक्तियों का क्वेरी करें जहाँ salary 10 से कम है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE salary < 10;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE salary < 10;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "58eb6ab2fb9c69becdbd_hi", "language": "hi", "backend": "supabase", "operation": "date_after", "question": "reservations में से वे पंक्तियाँ चुनें जिनके created_at में 2022-08-15 या उससे बाद की तारीख है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE created_at >= '2022-08-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE created_at >= '2022-08-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "42cda457b51f6f9d3056_hi", "language": "hi", "backend": "mysql", "operation": "eq", "question": "inspections से वे सभी पंक्तियाँ लाएँ जहाँ region का मान O'Reilly है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM inspections WHERE region = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE region = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0294353d6bde2cd5185c_hi", "language": "hi", "backend": "mysql", "operation": "lt", "question": "reservations से उन पंक्तियों का क्वेरी करें जहाँ cost 1 से कम है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations WHERE cost < 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations WHERE cost < 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9d0222589e9977ea3f57_hi", "language": "hi", "backend": "mysql", "operation": "lte", "question": "reservations से वे पंक्तियां खोजें जहां balance 1500 से बड़ा नहीं है।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM reservations WHERE balance <= 1500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE balance <= 1500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ba705525122da37a7146_hi", "language": "hi", "backend": "mysql", "operation": "sort_asc", "question": "reservations से सभी डेटा cost के बढ़ते हुए क्रम में लें और दिखाएं।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM reservations ORDER BY cost ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY cost ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d58041d2866c015bc923_hi", "language": "hi", "backend": "mysql", "operation": "lte", "question": "reservations से वे पंक्तियां खोजें जहां amount 100 से बड़ा नहीं है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations WHERE amount <= 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations WHERE amount <= 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b0f56c14239ba5d0227f_hi", "language": "hi", "backend": "mysql", "operation": "distinct", "question": "मुझे reservations में उपलब्ध region के विशिष्ट मान देखने हैं।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT DISTINCT region FROM reservations;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT DISTINCT region FROM reservations;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b9c176c07809f42564a0_hi", "language": "hi", "backend": "mysql", "operation": "top", "question": "शीर्ष 5 पंक्तियाँ inspections से प्राप्त करें, amount के आधार पर उच्चतम पहले।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount DESC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b45bf4e81303e2b4da0f_hi", "language": "hi", "backend": "mysql", "operation": "gt", "question": "specimens से उन पंक्तियों को प्रदर्शित करें जहाँ price 1 से ज्यादा है।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM specimens WHERE price > 1;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM specimens WHERE price > 1;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "6519cd640e9e4ab22a6e_hi", "language": "hi", "backend": "mysql", "operation": "gt", "question": "specimens से उन पंक्तियों को प्रदर्शित करें जहाँ score 20 से ज्यादा है।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens WHERE score > 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens WHERE score > 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "7fb4f6e9f1b68e05cfd9_hi", "language": "hi", "backend": "mysql", "operation": "sort_desc", "question": "reservations की पूरी सारणी salary के उच्चतम से निम्नतम क्रम में दिखाएँ।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM reservations ORDER BY salary DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY salary DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "41ea5651dda516a4ac30_hi", "language": "hi", "backend": "supabase", "operation": "date_after", "question": "reservations में से वे पंक्तियाँ चुनें जिनके order_date में 2023-12-15 या उससे बाद की तारीख है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT * FROM reservations WHERE order_date >= '2023-12-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT * FROM reservations WHERE order_date >= '2023-12-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9f8fa242b083f38bc5f8_hi", "language": "hi", "backend": "mysql", "operation": "month", "question": "मुझे inspections से वे डेटा पंक्तियां दें जहाँ order_date का महीना 4 है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM inspections WHERE MONTH(order_date) = 4;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE MONTH(order_date) = 4;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "c023a5b6f95173ae856d_hi", "language": "hi", "backend": "mysql", "operation": "gt", "question": "exhibits से उन पंक्तियों को प्रदर्शित करें जहाँ score 50 से ज्यादा है।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM exhibits WHERE score > 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE score > 50;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5890e52f983d16adbae7_hi", "language": "hi", "backend": "mysql", "operation": "top", "question": "शीर्ष 3 पंक्तियाँ inspections से प्राप्त करें, quantity के आधार पर उच्चतम पहले।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY quantity DESC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY quantity DESC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a29fdee3339da6711080_hi", "language": "hi", "backend": "mysql", "operation": "bottom", "question": "rating के बढ़ते क्रम में specimens की 20 सबसे कम पंक्तियां, सबसे कम से शुरू होकर, लें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM specimens ORDER BY rating ASC LIMIT 20;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM specimens ORDER BY rating ASC LIMIT 20;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "ae26d8175253b650bb6b_hi", "language": "hi", "backend": "supabase", "operation": "missing_value", "question": "region के आधार पर specimens पर एक फ़िल्टर निष्पादित करें, लेकिन मैंने कोई मान निर्दिष्ट नहीं किया है।", "expected": {"action": "clarify", "question": "region का कौन सा मान चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"region का कौन सा मान चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "2f545db764e4c6cd353d_hi", "language": "hi", "backend": "mysql", "operation": "gt", "question": "exhibits से उन पंक्तियों को प्रदर्शित करें जहाँ salary 0 से ज्यादा है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM exhibits WHERE salary > 0;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE salary > 0;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "bfb3f8261ddb2fc2bf28_hi", "language": "hi", "backend": "mysql", "operation": "bottom", "question": "price के बढ़ते क्रम में specimens की 3 सबसे कम पंक्तियां, सबसे कम से शुरू होकर, लें।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens ORDER BY price ASC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM specimens ORDER BY price ASC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "dc59f00f50b6c8f7bc16_hi", "language": "hi", "backend": "mysql", "operation": "between", "question": "exhibits से वे सभी आइटम चुनें जहाँ amount कॉलम का मान 20 और 520 के बीच है, समावेशी।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM exhibits WHERE amount BETWEEN 20 AND 520;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": false, "success": false, "error": "'sql'"}} +{"id": "c01aaa64706cccb8ac54_hi", "language": "hi", "backend": "mysql", "operation": "between", "question": "specimens से वे सभी आइटम चुनें जहाँ balance कॉलम का मान 0 और 50 के बीच है, समावेशी।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM specimens WHERE balance BETWEEN 0 AND 50;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"specimens\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": false, "success": false, "error": "'sql'"}} +{"id": "804d318fe46a0e664840_hi", "language": "hi", "backend": "mysql", "operation": "sort_asc", "question": "reservations से सभी डेटा quantity के बढ़ते हुए क्रम में लें और दिखाएं।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations ORDER BY quantity ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY quantity ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4e9184b509f43a710021_hi", "language": "hi", "backend": "mysql", "operation": "sort_desc", "question": "reservations की पूरी सारणी balance के उच्चतम से निम्नतम क्रम में दिखाएँ।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT * FROM reservations ORDER BY balance DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY balance DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2e9e0c695ac50e3982d7_hi", "language": "hi", "backend": "mysql", "operation": "month", "question": "मुझे exhibits से वे डेटा पंक्तियां दें जहाँ order_date का महीना 12 है।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits WHERE MONTH(order_date) = 12;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE MONTH(order_date) = 12;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "b10e3bcce6871b79df41_hi", "language": "hi", "backend": "mysql", "operation": "sort_asc", "question": "reservations से सभी डेटा price के बढ़ते हुए क्रम में लें और दिखाएं।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM reservations ORDER BY price ASC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY price ASC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "d1f372f544fa10f6df1b_hi", "language": "hi", "backend": "supabase", "operation": "join", "question": "कृपया reservations.group_id को reservations_groups.id से जोड़ें और reservations.title तथा reservations_groups.label प्राप्त करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT reservations.title, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT reservations.title, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "5f0b5d842746b96c126b_hi", "language": "hi", "backend": "mysql", "operation": "between", "question": "exhibits से वे सभी आइटम चुनें जहाँ price कॉलम का मान 1000 और 1100 के बीच है, समावेशी।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM exhibits WHERE price BETWEEN 1000 AND 1100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": false, "success": false, "error": "'sql'"}} +{"id": "8f9ab277b699f2cbe988_hi", "language": "hi", "backend": "mysql", "operation": "sort_desc", "question": "reservations की पूरी सारणी score के उच्चतम से निम्नतम क्रम में दिखाएँ।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM reservations ORDER BY score DESC;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM reservations ORDER BY score DESC;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "0d94a075cd7e5759bdfa_hi", "language": "hi", "backend": "mysql", "operation": "between", "question": "exhibits से वे सभी आइटम चुनें जहाँ rating कॉलम का मान 10 और 110 के बीच है, समावेशी।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM exhibits WHERE rating BETWEEN 10 AND 110;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.describe_table\",\"arguments\":{\"table\":\"exhibits\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": false, "success": false, "error": "'query'"}} +{"id": "bd760f1ca7aff546e852_hi", "language": "hi", "backend": "mysql", "operation": "bottom", "question": "balance के बढ़ते क्रम में exhibits की 5 सबसे कम पंक्तियां, सबसे कम से शुरू होकर, लें।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT * FROM exhibits ORDER BY balance ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits ORDER BY balance ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "139a72e10281049d1fed_hi", "language": "hi", "backend": "mysql", "operation": "join", "question": "कृपया specimens.group_id को specimens_groups.id से जोड़ें और specimens.name तथा specimens_groups.label प्राप्त करें।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT specimens.name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT specimens.name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4e4b83d8af9eb4c5efc8_hi", "language": "hi", "backend": "mysql", "operation": "bottom", "question": "amount के बढ़ते क्रम में inspections की 5 सबसे कम पंक्तियां, सबसे कम से शुरू होकर, लें।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT * FROM inspections ORDER BY amount ASC LIMIT 5;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT * FROM inspections ORDER BY amount ASC LIMIT 5;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "523a83d6ca07c1a1334a_hi", "language": "hi", "backend": "supabase", "operation": "missing_value", "question": "department के आधार पर exhibits पर एक फ़िल्टर निष्पादित करें, लेकिन मैंने कोई मान निर्दिष्ट नहीं किया है।", "expected": {"action": "clarify", "question": "department का कौन सा मान चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"department का कौन सा मान चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "5d05a2e11d5abab8423e_hi", "language": "hi", "backend": "supabase", "operation": "missing_value", "question": "region के आधार पर inspections पर एक फ़िल्टर निष्पादित करें, लेकिन मैंने कोई मान निर्दिष्ट नहीं किया है।", "expected": {"action": "clarify", "question": "region का कौन सा मान चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"region का कौन सा मान चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "fd9cafbb7e6e0ce69d06_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "join", "question": "output reservations.item_name and reservations_groups.label by joining reservations.group_id to reservations_groups.id", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "8a2e6d77aa1c3a517ecb_hi", "language": "hi", "backend": "mysql", "operation": "having", "question": "कृपया exhibits में 100 से अधिक पंक्तियों वाले category समूहों और उनकी गणना को प्रदर्शित करें।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT category, COUNT(*) FROM exhibits GROUP BY category HAVING COUNT(*) > 100;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT category, COUNT(*) FROM exhibits GROUP BY category HAVING COUNT(*) > 100;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "fd9cafbb7e6e0ce69d06_en", "language": "en", "backend": "mysql", "operation": "join", "question": "Extract reservations.item_name and reservations_groups.label by joining reservations.group_id to reservations_groups.id.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "844b55044ac74adf4493_hi", "language": "hi", "backend": "supabase", "operation": "join", "question": "कृपया reservations.group_id को reservations_groups.id से जोड़ें और reservations.item_name तथा reservations_groups.label प्राप्त करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_reservations", "query": "SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_reservations\",\"query\":\"SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "9956c110975a38c3d293_hi", "language": "hi", "backend": "mysql", "operation": "top", "question": "शीर्ष 3 पंक्तियाँ reservations से प्राप्त करें, cost के आधार पर उच्चतम पहले।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM reservations ORDER BY cost DESC LIMIT 3;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM reservations ORDER BY cost DESC LIMIT 3;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "00c24a7cc97ee9debf36_hi", "language": "hi", "backend": "mysql", "operation": "contains", "question": "exhibits से वे पंक्तियाँ दिखाएं जिनमें title pending के समान है, मामला अंतर नजरअंदाज करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT * FROM exhibits WHERE LOWER(title) LIKE LOWER('%pending%');"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT * FROM exhibits WHERE LOWER(title) LIKE LOWER('%pending%');\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "fd9cafbb7e6e0ce69d06_hinglish", "language": "hinglish", "backend": "mysql", "operation": "join", "question": "Kripya reservations.group_id ko reservations_groups.id se join karke reservations.item_name aur reservations_groups.label retrieve karo.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "307d5b7bd93949f7381a_hi", "language": "hi", "backend": "mysql", "operation": "join", "question": "कृपया specimens.group_id को specimens_groups.id से जोड़ें और specimens.customer_name तथा specimens_groups.label प्राप्त करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "53220f60789b4f5e4046_hi", "language": "hi", "backend": "mysql", "operation": "join", "question": "कृपया specimens.group_id को specimens_groups.id से जोड़ें और specimens.customer_name तथा specimens_groups.label प्राप्त करें।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT specimens.customer_name, specimens_groups.label FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f97582c94f2100eb1007_hi", "language": "hi", "backend": "mysql", "operation": "having", "question": "कृपया exhibits में 500 से अधिक पंक्तियों वाले department समूहों और उनकी गणना को प्रदर्शित करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 500;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT department, COUNT(*) FROM exhibits GROUP BY department HAVING COUNT(*) > 500;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a7675c74645d63d69aac_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "join_filter", "question": "show me reservations.description for reservations entries where reservations.group_id links to reservations_groups.id and reservations_groups.label is pending", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'pending';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'pending';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a7675c74645d63d69aac_hinglish", "language": "hinglish", "backend": "mysql", "operation": "join_filter", "question": "Please reservations.description dikhao jahan reservations reservations.group_id ke through reservations_groups se juda hai aur reservations_groups.label pending hai.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'pending';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'pending';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1b32b024d0c001fd9ae4_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "join_filter", "question": "show me reservations.description for reservations entries where reservations.group_id links to reservations_groups.id and reservations_groups.label is O'Reilly", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1b32b024d0c001fd9ae4_hinglish", "language": "hinglish", "backend": "mysql", "operation": "join_filter", "question": "Please reservations.description dikhao jahan reservations reservations.group_id ke through reservations_groups se juda hai aur reservations_groups.label O'Reilly hai.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a30eeaf286c79c95a1cd_hi", "language": "hi", "backend": "supabase", "operation": "join_filter", "question": "कृपया inspections.item_name दिखाएँ जहाँ inspections inspections.group_id के माध्यम से inspections_groups से जुड़ा है और inspections_groups.label O'Reilly है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"project_id": "demo_inspections", "query": "SELECT inspections.item_name FROM inspections JOIN inspections_groups ON inspections.group_id = inspections_groups.id WHERE inspections_groups.label = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"project_id\":\"demo_inspections\",\"query\":\"SELECT inspections.item_name FROM inspections JOIN inspections_groups ON inspections.group_id = inspections_groups.id WHERE inspections_groups.label = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a7675c74645d63d69aac_en", "language": "en", "backend": "mysql", "operation": "join_filter", "question": "Give me reservations.description for reservations entries where reservations.group_id points to reservations_groups.id and reservations_groups.label is pending.", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'pending';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'pending';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "30a7ec8a99d4ee6b6d0e_hi", "language": "hi", "backend": "mysql", "operation": "date_after", "question": "inspections में से वे पंक्तियाँ चुनें जिनके order_date में 2026-12-15 या उससे बाद की तारीख है।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"sql": "SELECT * FROM inspections WHERE order_date >= '2026-12-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"sql\":\"SELECT * FROM inspections WHERE order_date >= '2026-12-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1b32b024d0c001fd9ae4_en", "language": "en", "backend": "mysql", "operation": "join_filter", "question": "Give me reservations.description for reservations entries where reservations.group_id points to reservations_groups.id and reservations_groups.label is O'Reilly.", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "2cf634192448e68935c1_hi", "language": "hi", "backend": "mysql", "operation": "final", "question": "डेटाबेस टूल ने 5 पंक्तियां वापस कीं। कितनी पंक्तियां लौटीं?", "expected": {"action": "answer", "text": "टूल ने 5 पंक्तियाँ लौटाईं।"}, "output": "{\"action\":\"answer\",\"text\":\"टूल ने 5 पंक्तियाँ लौटाईं।\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "a59c6e6bcec70a1667e5_hi", "language": "hi", "backend": "mysql", "operation": "final", "question": "डेटाबेस टूल ने 35 पंक्तियां वापस कीं। कितनी पंक्तियां लौटीं?", "expected": {"action": "answer", "text": "टूल ने 35 पंक्तियाँ लौटाईं।"}, "output": "{\"action\":\"answer\",\"text\":\"टूल ने 35 पंक्तियाँ लौटाईं।\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "01f4f9527d44f09ab6b6_hi", "language": "hi", "backend": "mysql", "operation": "having", "question": "कृपया reservations में 200 से अधिक पंक्तियों वाले kind समूहों और उनकी गणना को प्रदर्शित करें।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT kind, COUNT(*) FROM reservations GROUP BY kind HAVING COUNT(*) > 200;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT kind, COUNT(*) FROM reservations GROUP BY kind HAVING COUNT(*) > 200;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "55c82b78d80d85926577_hi", "language": "hi", "backend": "mysql", "operation": "date_after", "question": "exhibits में से वे पंक्तियाँ चुनें जिनके start_date में 2026-09-15 या उससे बाद की तारीख है।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT * FROM exhibits WHERE start_date >= '2026-09-15';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT * FROM exhibits WHERE start_date >= '2026-09-15';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4a37ab647af0a5a5cb30_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "join_filter", "question": "show me reservations.display_name for reservations entries where reservations.group_id links to reservations_groups.id and reservations_groups.label is Electronics", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT reservations.display_name FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'Electronics';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT reservations.display_name FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4a37ab647af0a5a5cb30_hinglish", "language": "hinglish", "backend": "mysql", "operation": "join_filter", "question": "Please reservations.display_name dikhao jahan reservations reservations.group_id ke through reservations_groups se juda hai aur reservations_groups.label Electronics hai.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT reservations.display_name FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'Electronics';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT reservations.display_name FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4a37ab647af0a5a5cb30_en", "language": "en", "backend": "mysql", "operation": "join_filter", "question": "Give me reservations.display_name for reservations entries where reservations.group_id points to reservations_groups.id and reservations_groups.label is Electronics.", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT reservations.display_name FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'Electronics';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT reservations.display_name FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "63fa94106f0cf26f7da2_hi", "language": "hi", "backend": "mysql", "operation": "join_filter", "question": "कृपया specimens.title दिखाएँ जहाँ specimens specimens.group_id के माध्यम से specimens_groups से जुड़ा है और specimens_groups.label North है।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"query": "SELECT specimens.title FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id WHERE specimens_groups.label = 'North';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"query\":\"SELECT specimens.title FROM specimens JOIN specimens_groups ON specimens.group_id = specimens_groups.id WHERE specimens_groups.label = 'North';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "f55358178d003de7ba56_hi", "language": "hi", "backend": "mysql", "operation": "final", "question": "डेटाबेस टूल ने 133 पंक्तियां वापस कीं। कितनी पंक्तियां लौटीं?", "expected": {"action": "answer", "text": "टूल ने 133 पंक्तियाँ लौटाईं।"}, "output": "{\"action\":\"answer\",\"text\":\"टूल ने 133 पंक्तियाँ लौटाईं।\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "40ffef08b12e4a3276e3_hi", "language": "hi", "backend": "mysql", "operation": "final", "question": "डेटाबेस टूल ने 116 पंक्तियां वापस कीं। कितनी पंक्तियां लौटीं?", "expected": {"action": "answer", "text": "टूल ने 116 पंक्तियाँ लौटाईं।"}, "output": "{\"action\":\"answer\",\"text\":\"टूल ने 116 पंक्तियाँ लौटाईं।\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "5f07a4c7c387316664f1_hi", "language": "hi", "backend": "mysql", "operation": "missing_value", "question": "department के आधार पर exhibits पर एक फ़िल्टर निष्पादित करें, लेकिन मैंने कोई मान निर्दिष्ट नहीं किया है।", "expected": {"action": "clarify", "question": "department का कौन सा मान चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"department का कौन सा मान चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "3fdbd39c598af9b9ea10_hi", "language": "hi", "backend": "mysql", "operation": "missing_value", "question": "status के आधार पर reservations पर एक फ़िल्टर निष्पादित करें, लेकिन मैंने कोई मान निर्दिष्ट नहीं किया है।", "expected": {"action": "clarify", "question": "status का कौन सा मान चाहिए?"}, "output": "{\"action\":\"clarify\",\"question\":\"status का कौन सा मान चाहिए?\"}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": null, "arguments_exact": null, "sql_equivalent": null, "success": true, "text_exact": true}} +{"id": "fd9cafbb7e6e0ce69d06_hi", "language": "hi", "backend": "mysql", "operation": "join", "question": "कृपया reservations.group_id को reservations_groups.id से जोड़ें और reservations.item_name तथा reservations_groups.label प्राप्त करें।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"query": "SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"query\":\"SELECT reservations.item_name, reservations_groups.label FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id;\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "a7675c74645d63d69aac_hi", "language": "hi", "backend": "mysql", "operation": "join_filter", "question": "कृपया reservations.description दिखाएँ जहाँ reservations reservations.group_id के माध्यम से reservations_groups से जुड़ा है और reservations_groups.label pending है।", "expected": {"action": "call", "name": "warehouse.execute_sql", "arguments": {"sql": "SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'pending';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.execute_sql\",\"arguments\":{\"sql\":\"SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'pending';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "1b32b024d0c001fd9ae4_hi", "language": "hi", "backend": "mysql", "operation": "join_filter", "question": "कृपया reservations.description दिखाएँ जहाँ reservations reservations.group_id के माध्यम से reservations_groups से जुड़ा है और reservations_groups.label O'Reilly है।", "expected": {"action": "call", "name": "warehouse.query", "arguments": {"query": "SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'O''Reilly';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.query\",\"arguments\":{\"query\":\"SELECT reservations.description FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'O''Reilly';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "4a37ab647af0a5a5cb30_hi", "language": "hi", "backend": "mysql", "operation": "join_filter", "question": "कृपया reservations.display_name दिखाएँ जहाँ reservations reservations.group_id के माध्यम से reservations_groups से जुड़ा है और reservations_groups.label Electronics है।", "expected": {"action": "call", "name": "warehouse.run_query", "arguments": {"sql": "SELECT reservations.display_name FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'Electronics';"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.run_query\",\"arguments\":{\"sql\":\"SELECT reservations.display_name FROM reservations JOIN reservations_groups ON reservations.group_id = reservations_groups.id WHERE reservations_groups.label = 'Electronics';\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": true, "success": true}} +{"id": "88b478ce80eed61bdfb4_hinglish", "language": "hinglish", "backend": "supabase", "operation": "weather", "question": "Jaipur ka mausam celsius mein check karo.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Jaipur", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "6ba167225d0a0648e865_hinglish", "language": "hinglish", "backend": "supabase", "operation": "weather", "question": "Lucknow ka mausam celsius mein check karo.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Lucknow", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Lucknow\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "d84c7c968e50589a3321_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "search", "question": "look up API authentication in the doc", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "API authentication"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"API authentication\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "d84c7c968e50589a3321_hinglish", "language": "hinglish", "backend": "supabase", "operation": "search", "question": "Documentation ko search karke API authentication nikalo.", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "API authentication"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"API authentication\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "88b478ce80eed61bdfb4_en", "language": "en", "backend": "supabase", "operation": "weather", "question": "Access the weather data for Jaipur in celsius.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Jaipur", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "88b478ce80eed61bdfb4_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "weather", "question": "Get the weather status for Jaipur using celsius.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Jaipur", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "6ba167225d0a0648e865_en", "language": "en", "backend": "supabase", "operation": "weather", "question": "Access the weather data for Lucknow in celsius.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Lucknow", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Lucknow\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "6ba167225d0a0648e865_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "weather", "question": "Get the weather status for Lucknow using celsius.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Lucknow", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Lucknow\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "d84c7c968e50589a3321_en", "language": "en", "backend": "supabase", "operation": "search", "question": "Search through the documentation to find API authentication.", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "API authentication"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"API authentication\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "266ea8aafdf18d7c9212_en", "language": "en", "backend": "supabase", "operation": "ticket", "question": "Access the data for ticket TKT-6369.", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-6369"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6369\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "cca31e425efaaadf4c97_en", "language": "en", "backend": "supabase", "operation": "ticket", "question": "Access the data for ticket TKT-9184.", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-9184"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-9184\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "266ea8aafdf18d7c9212_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "ticket", "question": "i need to look at support ticket TKT-6369", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-6369"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6369\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "266ea8aafdf18d7c9212_hinglish", "language": "hinglish", "backend": "supabase", "operation": "ticket", "question": "support ticket TKT-6369 ki details show karo.", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-6369"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6369\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "cca31e425efaaadf4c97_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "ticket", "question": "i need to look at support ticket TKT-9184", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-9184"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-9184\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "cca31e425efaaadf4c97_hinglish", "language": "hinglish", "backend": "supabase", "operation": "ticket", "question": "support ticket TKT-9184 ki details show karo.", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-9184"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-9184\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "b6a1ad15475eea3c8bfb_hinglish", "language": "hinglish", "backend": "supabase", "operation": "read_file", "question": "/reports/sales.csv par file padh kar data lao.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/reports/sales.csv"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/reports/sales.csv\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "88b478ce80eed61bdfb4_hi", "language": "hi", "backend": "supabase", "operation": "weather", "question": "Jaipur का मौसम celsius में पता करें।", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Jaipur", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "6ba167225d0a0648e865_hi", "language": "hi", "backend": "supabase", "operation": "weather", "question": "Lucknow का मौसम celsius में पता करें।", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Lucknow", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Lucknow\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "b6a1ad15475eea3c8bfb_en", "language": "en", "backend": "supabase", "operation": "read_file", "question": "Retrieve the data by reading the file at /reports/sales.csv.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/reports/sales.csv"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/reports/sales.csv\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "b6a1ad15475eea3c8bfb_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "read_file", "question": "Get the data from the file at /reports/sales.csv by reading it.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/reports/sales.csv"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/reports/sales.csv\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "d84c7c968e50589a3321_hi", "language": "hi", "backend": "supabase", "operation": "search", "question": "दस्तावेज़ों में API authentication का उल्लेख खोजें।", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "API authentication"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"API authentication\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "266ea8aafdf18d7c9212_hi", "language": "hi", "backend": "supabase", "operation": "ticket", "question": "सपोर्ट टिकट TKT-6369 की जानकारी दिखाई दीजिए।", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-6369"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6369\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "4e42bbd40467b2b8c27f_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "search", "question": "look up SQL joins in the doc", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "SQL joins"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"SQL joins\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "4e42bbd40467b2b8c27f_hinglish", "language": "hinglish", "backend": "supabase", "operation": "search", "question": "Documentation ko search karke SQL joins nikalo.", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "SQL joins"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"SQL joins\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "cca31e425efaaadf4c97_hi", "language": "hi", "backend": "supabase", "operation": "ticket", "question": "सपोर्ट टिकट TKT-9184 की जानकारी दिखाई दीजिए।", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-9184"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-9184\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "635a24ec4bc42b67226a_hinglish", "language": "hinglish", "backend": "supabase", "operation": "weather", "question": "Jaipur ka mausam celsius mein check karo.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Jaipur", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "4e42bbd40467b2b8c27f_en", "language": "en", "backend": "supabase", "operation": "search", "question": "Search through the documentation to find SQL joins.", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "SQL joins"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"SQL joins\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "b6a1ad15475eea3c8bfb_hi", "language": "hi", "backend": "supabase", "operation": "read_file", "question": "/reports/sales.csv पर स्थित फ़ाइल को पढ़कर डेटा प्राप्त करें।", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/reports/sales.csv"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/reports/sales.csv\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "635a24ec4bc42b67226a_en", "language": "en", "backend": "supabase", "operation": "weather", "question": "Access the weather data for Jaipur in celsius.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Jaipur", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "635a24ec4bc42b67226a_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "weather", "question": "Get the weather status for Jaipur using celsius.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Jaipur", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "8a696e3c3522dd69d5ac_en", "language": "en", "backend": "supabase", "operation": "ticket", "question": "Access the data for ticket TKT-4486.", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-4486"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-4486\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "3dbbf63e641d9abf49db_en", "language": "en", "backend": "supabase", "operation": "ticket", "question": "Access the data for ticket TKT-9979.", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-9979"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-9979\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "8a696e3c3522dd69d5ac_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "ticket", "question": "i need to look at support ticket TKT-4486", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-4486"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-4486\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "8a696e3c3522dd69d5ac_hinglish", "language": "hinglish", "backend": "supabase", "operation": "ticket", "question": "support ticket TKT-4486 ki details show karo.", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-4486"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-4486\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "3dbbf63e641d9abf49db_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "ticket", "question": "i need to look at support ticket TKT-9979", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-9979"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-9979\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "3dbbf63e641d9abf49db_hinglish", "language": "hinglish", "backend": "supabase", "operation": "ticket", "question": "support ticket TKT-9979 ki details show karo.", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-9979"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-9979\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "89c9ecdcb7242ef71d8b_en", "language": "en", "backend": "supabase", "operation": "ticket", "question": "Access the data for ticket TKT-5087.", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-5087"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-5087\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "89c9ecdcb7242ef71d8b_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "ticket", "question": "i need to look at support ticket TKT-5087", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-5087"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-5087\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "89c9ecdcb7242ef71d8b_hinglish", "language": "hinglish", "backend": "supabase", "operation": "ticket", "question": "support ticket TKT-5087 ki details show karo.", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-5087"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-5087\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "f73e21e0eb10d6617995_hinglish", "language": "hinglish", "backend": "supabase", "operation": "read_file", "question": "/config/app.json par file padh kar data lao.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/config/app.json"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "635a24ec4bc42b67226a_hi", "language": "hi", "backend": "supabase", "operation": "weather", "question": "Jaipur का मौसम celsius में पता करें।", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Jaipur", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Jaipur\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "0161f123d136665478c9_hinglish", "language": "hinglish", "backend": "supabase", "operation": "read_file", "question": "/notes/setup.txt par file padh kar data lao.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "f73e21e0eb10d6617995_en", "language": "en", "backend": "supabase", "operation": "read_file", "question": "Retrieve the data by reading the file at /config/app.json.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/config/app.json"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "0161f123d136665478c9_en", "language": "en", "backend": "supabase", "operation": "read_file", "question": "Retrieve the data by reading the file at /notes/setup.txt.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "f73e21e0eb10d6617995_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "read_file", "question": "Get the data from the file at /config/app.json by reading it.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/config/app.json"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "0161f123d136665478c9_noisy_en", "language": "noisy_en", "backend": "supabase", "operation": "read_file", "question": "Get the data from the file at /notes/setup.txt by reading it.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "4e42bbd40467b2b8c27f_hi", "language": "hi", "backend": "supabase", "operation": "search", "question": "दस्तावेज़ों में SQL joins का उल्लेख खोजें।", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "SQL joins"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"SQL joins\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "531ed6009fed93af93ef_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "search", "question": "look up indexes in the doc", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "indexes"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.search_docs\",\"arguments\":{\"query\":\"indexes\"}}", "metrics": {"json_valid": true, "schema_valid": false, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false, "error": "Unknown tool"}} +{"id": "531ed6009fed93af93ef_hinglish", "language": "hinglish", "backend": "mysql", "operation": "search", "question": "Documentation ko search karke indexes nikalo.", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "indexes"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.search_docs\",\"arguments\":{\"query\":\"indexes\"}}", "metrics": {"json_valid": true, "schema_valid": false, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false, "error": "Unknown tool"}} +{"id": "8c8ba16d1f868e7d81a9_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "search", "question": "look up indexes in the doc", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "indexes"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"indexes\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "8c8ba16d1f868e7d81a9_hinglish", "language": "hinglish", "backend": "mysql", "operation": "search", "question": "Documentation ko search karke indexes nikalo.", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "indexes"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"indexes\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "252c6053295d69a93c94_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "search", "question": "look up database backups in the doc", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "database backups"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "252c6053295d69a93c94_hinglish", "language": "hinglish", "backend": "mysql", "operation": "search", "question": "Documentation ko search karke database backups nikalo.", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "database backups"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "531ed6009fed93af93ef_en", "language": "en", "backend": "mysql", "operation": "search", "question": "Search through the documentation to find indexes.", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "indexes"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.search_docs\",\"arguments\":{\"query\":\"indexes\"}}", "metrics": {"json_valid": true, "schema_valid": false, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false, "error": "Unknown tool"}} +{"id": "8c8ba16d1f868e7d81a9_en", "language": "en", "backend": "mysql", "operation": "search", "question": "Search through the documentation to find indexes.", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "indexes"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"indexes\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "252c6053295d69a93c94_en", "language": "en", "backend": "mysql", "operation": "search", "question": "Search through the documentation to find database backups.", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "database backups"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "5ebc52b3d20bde2076b3_en", "language": "en", "backend": "mysql", "operation": "ticket", "question": "Access the data for ticket TKT-6124.", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-6124"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6124\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "754691bd1eea25d41ccf_hinglish", "language": "hinglish", "backend": "mysql", "operation": "weather", "question": "Delhi ka mausam fahrenheit mein check karo.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "6600e72cfc86012212d0_hinglish", "language": "hinglish", "backend": "mysql", "operation": "weather", "question": "Delhi ka mausam fahrenheit mein check karo.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "b5839920055891fb68c8_hinglish", "language": "hinglish", "backend": "mysql", "operation": "weather", "question": "Lucknow ka mausam celsius mein check karo.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Lucknow", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Lucknow\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "5ebc52b3d20bde2076b3_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "ticket", "question": "i need to look at support ticket TKT-6124", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-6124"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6124\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "5ebc52b3d20bde2076b3_hinglish", "language": "hinglish", "backend": "mysql", "operation": "ticket", "question": "support ticket TKT-6124 ki details show karo.", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-6124"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6124\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "754691bd1eea25d41ccf_en", "language": "en", "backend": "mysql", "operation": "weather", "question": "Access the weather data for Delhi in fahrenheit.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "754691bd1eea25d41ccf_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "weather", "question": "Get the weather status for Delhi using fahrenheit.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "6600e72cfc86012212d0_en", "language": "en", "backend": "mysql", "operation": "weather", "question": "Access the weather data for Delhi in fahrenheit.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "6600e72cfc86012212d0_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "weather", "question": "Get the weather status for Delhi using fahrenheit.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "b5839920055891fb68c8_en", "language": "en", "backend": "mysql", "operation": "weather", "question": "Access the weather data for Lucknow in celsius.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Lucknow", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Lucknow\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "b5839920055891fb68c8_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "weather", "question": "Get the weather status for Lucknow using celsius.", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Lucknow", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Lucknow\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "adad513dde5be462c95d_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "search", "question": "look up row level security in the doc", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "row level security"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"row level security\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "adad513dde5be462c95d_hinglish", "language": "hinglish", "backend": "mysql", "operation": "search", "question": "Documentation ko search karke row level security nikalo.", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "row level security"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"row level security\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "adad513dde5be462c95d_en", "language": "en", "backend": "mysql", "operation": "search", "question": "Search through the documentation to find row level security.", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "row level security"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"row level security\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "6f3055c580b33340c854_hinglish", "language": "hinglish", "backend": "mysql", "operation": "read_file", "question": "/notes/setup.txt par file padh kar data lao.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "ae382a2348421f540f9e_hinglish", "language": "hinglish", "backend": "mysql", "operation": "read_file", "question": "/config/app.json par file padh kar data lao.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/config/app.json"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "754691bd1eea25d41ccf_hi", "language": "hi", "backend": "mysql", "operation": "weather", "question": "Delhi का मौसम fahrenheit में पता करें।", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "8a696e3c3522dd69d5ac_hi", "language": "hi", "backend": "supabase", "operation": "ticket", "question": "सपोर्ट टिकट TKT-4486 की जानकारी दिखाई दीजिए।", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-4486"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-4486\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "6600e72cfc86012212d0_hi", "language": "hi", "backend": "mysql", "operation": "weather", "question": "Delhi का मौसम fahrenheit में पता करें।", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Delhi", "unit": "fahrenheit"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Delhi\",\"unit\":\"fahrenheit\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "b5839920055891fb68c8_hi", "language": "hi", "backend": "mysql", "operation": "weather", "question": "Lucknow का मौसम celsius में पता करें।", "expected": {"action": "call", "name": "external.get_weather", "arguments": {"city": "Lucknow", "unit": "celsius"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_weather\",\"arguments\":{\"city\":\"Lucknow\",\"unit\":\"celsius\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "6f3055c580b33340c854_en", "language": "en", "backend": "mysql", "operation": "read_file", "question": "Retrieve the data by reading the file at /notes/setup.txt.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "13d062ce5950bde16f8f_hinglish", "language": "hinglish", "backend": "mysql", "operation": "read_file", "question": "/docs/README.md par file padh kar data lao.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/docs/README.md"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/docs/README.md\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "3dbbf63e641d9abf49db_hi", "language": "hi", "backend": "supabase", "operation": "ticket", "question": "सपोर्ट टिकट TKT-9979 की जानकारी दिखाई दीजिए।", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-9979"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-9979\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "ae382a2348421f540f9e_en", "language": "en", "backend": "mysql", "operation": "read_file", "question": "Retrieve the data by reading the file at /config/app.json.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/config/app.json"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "89c9ecdcb7242ef71d8b_hi", "language": "hi", "backend": "supabase", "operation": "ticket", "question": "सपोर्ट टिकट TKT-5087 की जानकारी दिखाई दीजिए।", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-5087"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-5087\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "f73e21e0eb10d6617995_hi", "language": "hi", "backend": "supabase", "operation": "read_file", "question": "/config/app.json पर स्थित फ़ाइल को पढ़कर डेटा प्राप्त करें।", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/config/app.json"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "6f3055c580b33340c854_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "read_file", "question": "Get the data from the file at /notes/setup.txt by reading it.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "13d062ce5950bde16f8f_en", "language": "en", "backend": "mysql", "operation": "read_file", "question": "Retrieve the data by reading the file at /docs/README.md.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/docs/README.md"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/docs/README.md\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "ae382a2348421f540f9e_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "read_file", "question": "Get the data from the file at /config/app.json by reading it.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/config/app.json"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "0161f123d136665478c9_hi", "language": "hi", "backend": "supabase", "operation": "read_file", "question": "/notes/setup.txt पर स्थित फ़ाइल को पढ़कर डेटा प्राप्त करें।", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "13d062ce5950bde16f8f_noisy_en", "language": "noisy_en", "backend": "mysql", "operation": "read_file", "question": "Get the data from the file at /docs/README.md by reading it.", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/docs/README.md"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/docs/README.md\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "531ed6009fed93af93ef_hi", "language": "hi", "backend": "mysql", "operation": "search", "question": "दस्तावेज़ों में indexes का उल्लेख खोजें।", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "indexes"}}, "output": "{\"action\":\"call\",\"name\":\"warehouse.search_docs\",\"arguments\":{\"query\":\"indexes\"}}", "metrics": {"json_valid": true, "schema_valid": false, "action_correct": true, "tool_correct": false, "arguments_exact": false, "sql_equivalent": null, "success": false, "error": "Unknown tool"}} +{"id": "8c8ba16d1f868e7d81a9_hi", "language": "hi", "backend": "mysql", "operation": "search", "question": "दस्तावेज़ों में indexes का उल्लेख खोजें।", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "indexes"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"indexes\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "252c6053295d69a93c94_hi", "language": "hi", "backend": "mysql", "operation": "search", "question": "दस्तावेज़ों में database backups का उल्लेख खोजें।", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "database backups"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"database backups\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "adad513dde5be462c95d_hi", "language": "hi", "backend": "mysql", "operation": "search", "question": "दस्तावेज़ों में row level security का उल्लेख खोजें।", "expected": {"action": "call", "name": "external.search_docs", "arguments": {"query": "row level security"}}, "output": "{\"action\":\"call\",\"name\":\"external.search_docs\",\"arguments\":{\"query\":\"row level security\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "5ebc52b3d20bde2076b3_hi", "language": "hi", "backend": "mysql", "operation": "ticket", "question": "सपोर्ट टिकट TKT-6124 की जानकारी दिखाई दीजिए।", "expected": {"action": "call", "name": "external.get_ticket", "arguments": {"ticket_id": "TKT-6124"}}, "output": "{\"action\":\"call\",\"name\":\"external.get_ticket\",\"arguments\":{\"ticket_id\":\"TKT-6124\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "6f3055c580b33340c854_hi", "language": "hi", "backend": "mysql", "operation": "read_file", "question": "/notes/setup.txt पर स्थित फ़ाइल को पढ़कर डेटा प्राप्त करें।", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/notes/setup.txt"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/notes/setup.txt\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "ae382a2348421f540f9e_hi", "language": "hi", "backend": "mysql", "operation": "read_file", "question": "/config/app.json पर स्थित फ़ाइल को पढ़कर डेटा प्राप्त करें।", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/config/app.json"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/config/app.json\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} +{"id": "13d062ce5950bde16f8f_hi", "language": "hi", "backend": "mysql", "operation": "read_file", "question": "/docs/README.md पर स्थित फ़ाइल को पढ़कर डेटा प्राप्त करें।", "expected": {"action": "call", "name": "external.read_file", "arguments": {"path": "/docs/README.md"}}, "output": "{\"action\":\"call\",\"name\":\"external.read_file\",\"arguments\":{\"path\":\"/docs/README.md\"}}", "metrics": {"json_valid": true, "schema_valid": true, "action_correct": true, "tool_correct": true, "arguments_exact": true, "sql_equivalent": null, "success": true}} diff --git a/evaluation/test-mac.summary.json b/evaluation/test-mac.summary.json new file mode 100644 index 0000000000000000000000000000000000000000..e300ed6c94d754ddc3af5595d062a402cb919b5b --- /dev/null +++ b/evaluation/test-mac.summary.json @@ -0,0 +1,1933 @@ +{ + "examples": 1196, + "seconds": 141.622967004776, + "decoding": "raw greedy, no repair, no teacher fallback", + "checkpoint_sha256": "9489b5cde69c11f64b3e7031182e8248aa83790f1ad4aeefbf99a4e5e152e692", + "sql_metric": "Compile against the supplied schema, then compare results on two generated SQLite fixtures after dialect adaptation; not native MySQL/PostgreSQL execution.", + "metrics": { + "json_valid": { + "correct": 1196, + "total": 1196, + "rate": 1.0 + }, + "schema_valid": { + "correct": 1191, + "total": 1196, + "rate": 0.995819397993311 + }, + "action_correct": { + "correct": 1192, + "total": 1196, + "rate": 0.9966555183946488 + }, + "tool_correct": { + "correct": 1088, + "total": 1100, + "rate": 0.9890909090909091 + }, + "arguments_exact": { + "correct": 1014, + "total": 1100, + "rate": 0.9218181818181819 + }, + "sql_equivalent": { + "correct": 896, + "total": 912, + "rate": 0.9824561403508771 + }, + "success": { + "correct": 1106, + "total": 1196, + "rate": 0.9247491638795987 + } + }, + "groups": { + "language:noisy_en": { + "json_valid": { + "correct": 299, + "total": 299, + "rate": 1.0 + }, + "schema_valid": { + "correct": 298, + "total": 299, + "rate": 0.9966555183946488 + }, + "action_correct": { + "correct": 299, + "total": 299, + "rate": 1.0 + }, + "tool_correct": { + "correct": 274, + "total": 275, + "rate": 0.9963636363636363 + }, + "arguments_exact": { + "correct": 252, + "total": 275, + "rate": 0.9163636363636364 + }, + "sql_equivalent": { + "correct": 228, + "total": 228, + "rate": 1.0 + }, + "success": { + "correct": 276, + "total": 299, + "rate": 0.9230769230769231 + } + }, + "backend:supabase": { + "json_valid": { + "correct": 588, + "total": 588, + "rate": 1.0 + }, + "schema_valid": { + "correct": 588, + "total": 588, + "rate": 1.0 + }, + "action_correct": { + "correct": 588, + "total": 588, + "rate": 1.0 + }, + "tool_correct": { + "correct": 541, + "total": 544, + "rate": 0.9944852941176471 + }, + "arguments_exact": { + "correct": 480, + "total": 544, + "rate": 0.8823529411764706 + }, + "sql_equivalent": { + "correct": 429, + "total": 436, + "rate": 0.9839449541284404 + }, + "success": { + "correct": 524, + "total": 588, + "rate": 0.891156462585034 + } + }, + "operation:missing_schema": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 20, + "total": 24, + "rate": 0.8333333333333334 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 20, + "total": 24, + "rate": 0.8333333333333334 + } + }, + "language:en": { + "json_valid": { + "correct": 299, + "total": 299, + "rate": 1.0 + }, + "schema_valid": { + "correct": 298, + "total": 299, + "rate": 0.9966555183946488 + }, + "action_correct": { + "correct": 297, + "total": 299, + "rate": 0.9933110367892977 + }, + "tool_correct": { + "correct": 274, + "total": 275, + "rate": 0.9963636363636363 + }, + "arguments_exact": { + "correct": 246, + "total": 275, + "rate": 0.8945454545454545 + }, + "sql_equivalent": { + "correct": 222, + "total": 228, + "rate": 0.9736842105263158 + }, + "success": { + "correct": 268, + "total": 299, + "rate": 0.8963210702341137 + } + }, + "language:hinglish": { + "json_valid": { + "correct": 299, + "total": 299, + "rate": 1.0 + }, + "schema_valid": { + "correct": 297, + "total": 299, + "rate": 0.9933110367892977 + }, + "action_correct": { + "correct": 297, + "total": 299, + "rate": 0.9933110367892977 + }, + "tool_correct": { + "correct": 273, + "total": 275, + "rate": 0.9927272727272727 + }, + "arguments_exact": { + "correct": 262, + "total": 275, + "rate": 0.9527272727272728 + }, + "sql_equivalent": { + "correct": 228, + "total": 228, + "rate": 1.0 + }, + "success": { + "correct": 284, + "total": 299, + "rate": 0.9498327759197325 + } + }, + "operation:sql_error": { + "json_valid": { + "correct": 20, + "total": 20, + "rate": 1.0 + }, + "schema_valid": { + "correct": 20, + "total": 20, + "rate": 1.0 + }, + "action_correct": { + "correct": 20, + "total": 20, + "rate": 1.0 + }, + "tool_correct": { + "correct": 20, + "total": 20, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 12, + "total": 20, + "rate": 0.6 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 12, + "total": 20, + "rate": 0.6 + } + }, + "backend:mysql": { + "json_valid": { + "correct": 608, + "total": 608, + "rate": 1.0 + }, + "schema_valid": { + "correct": 603, + "total": 608, + "rate": 0.9917763157894737 + }, + "action_correct": { + "correct": 604, + "total": 608, + "rate": 0.993421052631579 + }, + "tool_correct": { + "correct": 547, + "total": 556, + "rate": 0.9838129496402878 + }, + "arguments_exact": { + "correct": 534, + "total": 556, + "rate": 0.960431654676259 + }, + "sql_equivalent": { + "correct": 467, + "total": 476, + "rate": 0.9810924369747899 + }, + "success": { + "correct": 582, + "total": 608, + "rate": 0.9572368421052632 + } + }, + "operation:list_tables": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 12, + "total": 24, + "rate": 0.5 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 12, + "total": 24, + "rate": 0.5 + } + }, + "operation:all": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:count": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 23, + "total": 28, + "rate": 0.8214285714285714 + }, + "sql_equivalent": { + "correct": 23, + "total": 28, + "rate": 0.8214285714285714 + }, + "success": { + "correct": 23, + "total": 28, + "rate": 0.8214285714285714 + } + }, + "operation:avg": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 26, + "total": 28, + "rate": 0.9285714285714286 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 26, + "total": 28, + "rate": 0.9285714285714286 + } + }, + "operation:describe": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 23, + "total": 24, + "rate": 0.9583333333333334 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 23, + "total": 24, + "rate": 0.9583333333333334 + }, + "arguments_exact": { + "correct": 19, + "total": 24, + "rate": 0.7916666666666666 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 19, + "total": 24, + "rate": 0.7916666666666666 + } + }, + "operation:project": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:gte": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 26, + "total": 28, + "rate": 0.9285714285714286 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 26, + "total": 28, + "rate": 0.9285714285714286 + } + }, + "operation:min": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 24, + "total": 28, + "rate": 0.8571428571428571 + }, + "sql_equivalent": { + "correct": 24, + "total": 28, + "rate": 0.8571428571428571 + }, + "success": { + "correct": 24, + "total": 28, + "rate": 0.8571428571428571 + } + }, + "operation:null": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 26, + "total": 28, + "rate": 0.9285714285714286 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 26, + "total": 28, + "rate": 0.9285714285714286 + } + }, + "operation:write": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 20, + "total": 24, + "rate": 0.8333333333333334 + }, + "tool_correct": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "arguments_exact": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 20, + "total": 24, + "rate": 0.8333333333333334 + } + }, + "operation:or": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:count_eq": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 25, + "total": 28, + "rate": 0.8928571428571429 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 25, + "total": 28, + "rate": 0.8928571428571429 + } + }, + "operation:contains": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "success": { + "correct": 24, + "total": 24, + "rate": 1.0 + } + }, + "operation:sum": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 26, + "total": 28, + "rate": 0.9285714285714286 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 26, + "total": 28, + "rate": 0.9285714285714286 + } + }, + "operation:sort_desc": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 26, + "total": 28, + "rate": 0.9285714285714286 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 26, + "total": 28, + "rate": 0.9285714285714286 + } + }, + "operation:eq": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:project_eq": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "language:hi": { + "json_valid": { + "correct": 299, + "total": 299, + "rate": 1.0 + }, + "schema_valid": { + "correct": 298, + "total": 299, + "rate": 0.9966555183946488 + }, + "action_correct": { + "correct": 299, + "total": 299, + "rate": 1.0 + }, + "tool_correct": { + "correct": 267, + "total": 275, + "rate": 0.9709090909090909 + }, + "arguments_exact": { + "correct": 254, + "total": 275, + "rate": 0.9236363636363636 + }, + "sql_equivalent": { + "correct": 218, + "total": 228, + "rate": 0.956140350877193 + }, + "success": { + "correct": 278, + "total": 299, + "rate": 0.9297658862876255 + } + }, + "operation:year": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 26, + "total": 28, + "rate": 0.9285714285714286 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 26, + "total": 28, + "rate": 0.9285714285714286 + } + }, + "operation:bottom": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:distinct": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:ambiguous": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "arguments_exact": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 24, + "total": 24, + "rate": 1.0 + } + }, + "operation:max": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 26, + "total": 28, + "rate": 0.9285714285714286 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 26, + "total": 28, + "rate": 0.9285714285714286 + } + }, + "operation:group_count": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:sort_asc": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:not_null": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 26, + "total": 28, + "rate": 0.9285714285714286 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 26, + "total": 28, + "rate": 0.9285714285714286 + } + }, + "operation:lte": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:lt": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 26, + "total": 28, + "rate": 0.9285714285714286 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 26, + "total": 28, + "rate": 0.9285714285714286 + } + }, + "operation:group_sum": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:between": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 21, + "total": 28, + "rate": 0.75 + }, + "arguments_exact": { + "correct": 21, + "total": 28, + "rate": 0.75 + }, + "sql_equivalent": { + "correct": 21, + "total": 28, + "rate": 0.75 + }, + "success": { + "correct": 21, + "total": 28, + "rate": 0.75 + } + }, + "operation:month": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 24, + "total": 28, + "rate": 0.8571428571428571 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 24, + "total": 28, + "rate": 0.8571428571428571 + } + }, + "operation:and": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 26, + "total": 28, + "rate": 0.9285714285714286 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 26, + "total": 28, + "rate": 0.9285714285714286 + } + }, + "operation:having": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 20, + "total": 28, + "rate": 0.7142857142857143 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 20, + "total": 28, + "rate": 0.7142857142857143 + } + }, + "operation:missing_value": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "arguments_exact": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 24, + "total": 24, + "rate": 1.0 + } + }, + "operation:final": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "arguments_exact": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 24, + "total": 24, + "rate": 1.0 + } + }, + "operation:date_after": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:gt": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 26, + "total": 28, + "rate": 0.9285714285714286 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 26, + "total": 28, + "rate": 0.9285714285714286 + } + }, + "operation:top": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:join_filter": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "success": { + "correct": 24, + "total": 24, + "rate": 1.0 + } + }, + "operation:join": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "success": { + "correct": 24, + "total": 24, + "rate": 1.0 + } + }, + "operation:weather": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 24, + "total": 24, + "rate": 1.0 + } + }, + "operation:search": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 20, + "total": 24, + "rate": 0.8333333333333334 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 20, + "total": 24, + "rate": 0.8333333333333334 + }, + "arguments_exact": { + "correct": 20, + "total": 24, + "rate": 0.8333333333333334 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 20, + "total": 24, + "rate": 0.8333333333333334 + } + }, + "operation:ticket": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 24, + "total": 24, + "rate": 1.0 + } + }, + "operation:read_file": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 24, + "total": 24, + "rate": 1.0 + } + } + } +} \ No newline at end of file diff --git a/evaluation/validation.summary.json b/evaluation/validation.summary.json new file mode 100644 index 0000000000000000000000000000000000000000..4167787feca9a83c816013a86fac779d43757bd3 --- /dev/null +++ b/evaluation/validation.summary.json @@ -0,0 +1,1933 @@ +{ + "examples": 1200, + "seconds": 33.96494126319885, + "decoding": "raw greedy, no repair, no teacher fallback", + "checkpoint_sha256": "9489b5cde69c11f64b3e7031182e8248aa83790f1ad4aeefbf99a4e5e152e692", + "sql_metric": "Dialect parsing and equivalent results on two generated SQLite fixtures after dialect adaptation; not native MySQL/PostgreSQL execution.", + "metrics": { + "json_valid": { + "correct": 1200, + "total": 1200, + "rate": 1.0 + }, + "schema_valid": { + "correct": 1199, + "total": 1200, + "rate": 0.9991666666666666 + }, + "action_correct": { + "correct": 1193, + "total": 1200, + "rate": 0.9941666666666666 + }, + "tool_correct": { + "correct": 1103, + "total": 1104, + "rate": 0.9990942028985508 + }, + "arguments_exact": { + "correct": 1085, + "total": 1104, + "rate": 0.9827898550724637 + }, + "sql_equivalent": { + "correct": 899, + "total": 912, + "rate": 0.9857456140350878 + }, + "success": { + "correct": 1174, + "total": 1200, + "rate": 0.9783333333333334 + } + }, + "groups": { + "language:noisy_en": { + "json_valid": { + "correct": 300, + "total": 300, + "rate": 1.0 + }, + "schema_valid": { + "correct": 300, + "total": 300, + "rate": 1.0 + }, + "action_correct": { + "correct": 300, + "total": 300, + "rate": 1.0 + }, + "tool_correct": { + "correct": 276, + "total": 276, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 275, + "total": 276, + "rate": 0.9963768115942029 + }, + "sql_equivalent": { + "correct": 227, + "total": 228, + "rate": 0.9956140350877193 + }, + "success": { + "correct": 299, + "total": 300, + "rate": 0.9966666666666667 + } + }, + "backend:supabase": { + "json_valid": { + "correct": 604, + "total": 604, + "rate": 1.0 + }, + "schema_valid": { + "correct": 603, + "total": 604, + "rate": 0.9983443708609272 + }, + "action_correct": { + "correct": 601, + "total": 604, + "rate": 0.9950331125827815 + }, + "tool_correct": { + "correct": 559, + "total": 560, + "rate": 0.9982142857142857 + }, + "arguments_exact": { + "correct": 551, + "total": 560, + "rate": 0.9839285714285714 + }, + "sql_equivalent": { + "correct": 461, + "total": 468, + "rate": 0.9850427350427351 + }, + "success": { + "correct": 592, + "total": 604, + "rate": 0.9801324503311258 + } + }, + "operation:missing_schema": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 24, + "total": 24, + "rate": 1.0 + } + }, + "language:en": { + "json_valid": { + "correct": 300, + "total": 300, + "rate": 1.0 + }, + "schema_valid": { + "correct": 300, + "total": 300, + "rate": 1.0 + }, + "action_correct": { + "correct": 294, + "total": 300, + "rate": 0.98 + }, + "tool_correct": { + "correct": 276, + "total": 276, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 268, + "total": 276, + "rate": 0.9710144927536232 + }, + "sql_equivalent": { + "correct": 222, + "total": 228, + "rate": 0.9736842105263158 + }, + "success": { + "correct": 286, + "total": 300, + "rate": 0.9533333333333334 + } + }, + "language:hinglish": { + "json_valid": { + "correct": 300, + "total": 300, + "rate": 1.0 + }, + "schema_valid": { + "correct": 299, + "total": 300, + "rate": 0.9966666666666667 + }, + "action_correct": { + "correct": 300, + "total": 300, + "rate": 1.0 + }, + "tool_correct": { + "correct": 275, + "total": 276, + "rate": 0.9963768115942029 + }, + "arguments_exact": { + "correct": 268, + "total": 276, + "rate": 0.9710144927536232 + }, + "sql_equivalent": { + "correct": 223, + "total": 228, + "rate": 0.9780701754385965 + }, + "success": { + "correct": 292, + "total": 300, + "rate": 0.9733333333333334 + } + }, + "backend:mysql": { + "json_valid": { + "correct": 596, + "total": 596, + "rate": 1.0 + }, + "schema_valid": { + "correct": 596, + "total": 596, + "rate": 1.0 + }, + "action_correct": { + "correct": 592, + "total": 596, + "rate": 0.9932885906040269 + }, + "tool_correct": { + "correct": 544, + "total": 544, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 534, + "total": 544, + "rate": 0.9816176470588235 + }, + "sql_equivalent": { + "correct": 438, + "total": 444, + "rate": 0.9864864864864865 + }, + "success": { + "correct": 582, + "total": 596, + "rate": 0.9765100671140939 + } + }, + "operation:all": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:write": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 18, + "total": 24, + "rate": 0.75 + }, + "tool_correct": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "arguments_exact": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 18, + "total": 24, + "rate": 0.75 + } + }, + "operation:distinct": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:max": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:min": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:avg": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:group_count": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 25, + "total": 28, + "rate": 0.8928571428571429 + }, + "sql_equivalent": { + "correct": 25, + "total": 28, + "rate": 0.8928571428571429 + }, + "success": { + "correct": 25, + "total": 28, + "rate": 0.8928571428571429 + } + }, + "language:hi": { + "json_valid": { + "correct": 300, + "total": 300, + "rate": 1.0 + }, + "schema_valid": { + "correct": 300, + "total": 300, + "rate": 1.0 + }, + "action_correct": { + "correct": 299, + "total": 300, + "rate": 0.9966666666666667 + }, + "tool_correct": { + "correct": 276, + "total": 276, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 274, + "total": 276, + "rate": 0.9927536231884058 + }, + "sql_equivalent": { + "correct": 227, + "total": 228, + "rate": 0.9956140350877193 + }, + "success": { + "correct": 297, + "total": 300, + "rate": 0.99 + } + }, + "operation:sum": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 27, + "total": 28, + "rate": 0.9642857142857143 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 27, + "total": 28, + "rate": 0.9642857142857143 + } + }, + "operation:count_eq": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:count": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 22, + "total": 28, + "rate": 0.7857142857142857 + }, + "sql_equivalent": { + "correct": 22, + "total": 28, + "rate": 0.7857142857142857 + }, + "success": { + "correct": 22, + "total": 28, + "rate": 0.7857142857142857 + } + }, + "operation:sql_error": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 20, + "total": 24, + "rate": 0.8333333333333334 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 20, + "total": 24, + "rate": 0.8333333333333334 + } + }, + "operation:project": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 24, + "total": 28, + "rate": 0.8571428571428571 + }, + "sql_equivalent": { + "correct": 24, + "total": 28, + "rate": 0.8571428571428571 + }, + "success": { + "correct": 24, + "total": 28, + "rate": 0.8571428571428571 + } + }, + "operation:lt": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:top": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:group_sum": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:sort_desc": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:ambiguous": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "arguments_exact": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 24, + "total": 24, + "rate": 1.0 + } + }, + "operation:gte": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:eq": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:or": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:month": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:project_eq": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:contains": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "success": { + "correct": 24, + "total": 24, + "rate": 1.0 + } + }, + "operation:not_null": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:gt": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:lte": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:bottom": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:having": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:sort_asc": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:null": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:year": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:missing_value": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "arguments_exact": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 24, + "total": 24, + "rate": 1.0 + } + }, + "operation:and": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:date_after": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:final": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 23, + "total": 24, + "rate": 0.9583333333333334 + }, + "tool_correct": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "arguments_exact": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 23, + "total": 24, + "rate": 0.9583333333333334 + } + }, + "operation:list_tables": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 24, + "total": 24, + "rate": 1.0 + } + }, + "operation:describe": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 24, + "total": 24, + "rate": 1.0 + } + }, + "operation:join": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "success": { + "correct": 24, + "total": 24, + "rate": 1.0 + } + }, + "operation:join_filter": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "success": { + "correct": 24, + "total": 24, + "rate": 1.0 + } + }, + "operation:between": { + "json_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "schema_valid": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "action_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "tool_correct": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 28, + "total": 28, + "rate": 1.0 + }, + "success": { + "correct": 28, + "total": 28, + "rate": 1.0 + } + }, + "operation:search": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 24, + "total": 24, + "rate": 1.0 + } + }, + "operation:weather": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 24, + "total": 24, + "rate": 1.0 + } + }, + "operation:ticket": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "arguments_exact": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 24, + "total": 24, + "rate": 1.0 + } + }, + "operation:read_file": { + "json_valid": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "schema_valid": { + "correct": 23, + "total": 24, + "rate": 0.9583333333333334 + }, + "action_correct": { + "correct": 24, + "total": 24, + "rate": 1.0 + }, + "tool_correct": { + "correct": 23, + "total": 24, + "rate": 0.9583333333333334 + }, + "arguments_exact": { + "correct": 23, + "total": 24, + "rate": 0.9583333333333334 + }, + "sql_equivalent": { + "correct": 0, + "total": 0, + "rate": 0.0 + }, + "success": { + "correct": 23, + "total": 24, + "rate": 0.9583333333333334 + } + } + } +} \ No newline at end of file diff --git a/examples/context-mysql.json b/examples/context-mysql.json new file mode 100644 index 0000000000000000000000000000000000000000..4345edf0533ed7bc2432c59c505027a38dcaeecb --- /dev/null +++ b/examples/context-mysql.json @@ -0,0 +1,59 @@ +{ + "backend": "mysql", + "project_id": "demo", + "schema": [ + "CREATE TABLE customers (id INTEGER PRIMARY KEY, name TEXT, city TEXT, amount REAL);" + ], + "tools": [ + { + "name": "query", + "description": "Execute a read-only MySQL SELECT query.", + "inputSchema": { + "type": "object", + "properties": { + "query": { + "type": "string" + } + }, + "required": [ + "query" + ], + "additionalProperties": false + } + }, + { + "name": "list_tables", + "description": "List database tables and their columns.", + "inputSchema": { + "type": "object", + "properties": { + "schemas": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [], + "additionalProperties": false + } + }, + { + "name": "describe_table", + "description": "Inspect columns and types for one table.", + "inputSchema": { + "type": "object", + "properties": { + "table": { + "type": "string" + } + }, + "required": [ + "table" + ], + "additionalProperties": false + } + } + ], + "policy": "Read-only database access. Use provided tools. Ask when required information is missing." +} \ No newline at end of file diff --git a/examples/context-supabase.json b/examples/context-supabase.json new file mode 100644 index 0000000000000000000000000000000000000000..f4745259aa7492bf1d61eaeb1f17dc164e3e9c76 --- /dev/null +++ b/examples/context-supabase.json @@ -0,0 +1,43 @@ +{ + "backend": "supabase", + "project_id": "demo", + "schema": [ + "CREATE TABLE customers (id INTEGER PRIMARY KEY, name TEXT, city TEXT, amount REAL);" + ], + "tools": [ + { + "name": "execute_sql", + "description": "Execute a read-only PostgreSQL SELECT query.", + "inputSchema": { + "type": "object", + "properties": { + "query": { + "type": "string" + } + }, + "required": [ + "query" + ], + "additionalProperties": false + } + }, + { + "name": "list_tables", + "description": "List database tables and their columns.", + "inputSchema": { + "type": "object", + "properties": { + "schemas": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [], + "additionalProperties": false + } + } + ], + "policy": "Read-only database access. Use provided tools. Ask when required information is missing." +} \ No newline at end of file diff --git a/grounding-stats.json b/grounding-stats.json new file mode 100644 index 0000000000000000000000000000000000000000..640d25ddf91747916e19dbad82dfce2923a0ee04 --- /dev/null +++ b/grounding-stats.json @@ -0,0 +1,5 @@ +{ + "layout": 8826, + "runtime_names": 13635, + "city_language": 4800 +} \ No newline at end of file diff --git a/manifest.json b/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..7a0053506b86c68bb0ae5eaf1be28e538b8f027f --- /dev/null +++ b/manifest.json @@ -0,0 +1,279 @@ +{ + "parameters": 139738113, + "format": "Custom native PyTorch; see tinyquery/model.py, not a Transformers AutoModel checkpoint", + "random_initialization": true, + "files": { + "LICENSE": { + "bytes": 11544, + "sha256": "bbedc3fda3305820b977265f01b8619d87570a6739de3a5582c3464840f1e57a" + }, + "README.md": { + "bytes": 9919, + "sha256": "34cd941a5170b840cbfa0450a741561a42b0ee8fae4b9fa26031ffe1026ae5dd" + }, + "checkpoint-info.json": { + "bytes": 144, + "sha256": "9626ef405296d0772a9df05af70e96dddebcd1d0840d29180855b65887be589a" + }, + "config.json": { + "bytes": 170, + "sha256": "94054411ca3c62c3b500405c4ab8c45ecd6d713d88dc95f726908ca8b9411df1" + }, + "docs/architecture.md": { + "bytes": 4774, + "sha256": "497a2b76330e203dcbcd4e38fd9b03a1adcad2197e1e6bb545210b34fc3574ad" + }, + "docs/experiment.md": { + "bytes": 5988, + "sha256": "1bba1663c1c927ca91299dedcfc200055420677df357829fd5271bd1ba8fe953" + }, + "evaluation/baseline-manual.jsonl": { + "bytes": 64508, + "sha256": "7c45ebd7aa01711b10f0d15e47e1c46240c29acae0d70f8392693e7e33207020" + }, + "evaluation/baseline-manual.summary.json": { + "bytes": 19377, + "sha256": "6f7ac7ea1ab6e2b1b3297bb3393f6e4fd21caefa7962a2d3983662807b6fb527" + }, + "evaluation/baseline-test.jsonl": { + "bytes": 486708, + "sha256": "630ec167061e963e4557706fb74282dbd3d5ceb2e0951dd6cfc3f8c0b80cff47" + }, + "evaluation/baseline-test.summary.json": { + "bytes": 38746, + "sha256": "80bb5910474dc7103c73265d8268f4de9b40209a964b54c5a33d9345d5f0d638" + }, + "evaluation/development.summary.json": { + "bytes": 34207, + "sha256": "d9b17d2ab010df858674362a4cdf5275939d8728b594769637fa5b7954a1f2cd" + }, + "evaluation/gpu-final-console-metrics.json": { + "bytes": 2062, + "sha256": "1760d2862fe481bb65d8bfad8110541d315e740ec1546996e777b547f4291b6b" + }, + "evaluation/manual-mac.jsonl": { + "bytes": 98789, + "sha256": "6c028d9dbf90bea9dcd7daf8c4f529b149deaaabb780edcd764d391783865263" + }, + "evaluation/manual-mac.summary.json": { + "bytes": 19604, + "sha256": "ecb96cfa02aa55a196e55b3bb9bbb331729d5cc8d0a79b6dd5bfbba958cef75b" + }, + "evaluation/test-mac.jsonl": { + "bytes": 794897, + "sha256": "6006020aefb90eecd3fa282cf3fceb445e32fc27bfe11bd967ce8cd7a86702d8" + }, + "evaluation/test-mac.summary.json": { + "bytes": 38589, + "sha256": "17b685763d7a20d3d3256a6264eacf3f569e461cf46eb68ee9160e04ecbe8817" + }, + "evaluation/validation.summary.json": { + "bytes": 38081, + "sha256": "86a6a6e36522decadd3e93be9bcdc9435c9f619fd16290cfa78542001358d354" + }, + "examples/context-mysql.json": { + "bytes": 1366, + "sha256": "49fca0302ceb7e5199997a905c309533768cc4e0310be62b092d40dfe19d3209" + }, + "examples/context-supabase.json": { + "bytes": 1027, + "sha256": "d54c9e69716303ce5a57d79b75f8f26499c6fb6131663ff168e9d5697d9122ea" + }, + "grounding-stats.json": { + "bytes": 71, + "sha256": "307e4a794c8257fd4fab4040994070560fd3f60a8f73143a7a044c986ea4d545" + }, + "model.safetensors": { + "bytes": 279483882, + "sha256": "9489b5cde69c11f64b3e7031182e8248aa83790f1ad4aeefbf99a4e5e152e692" + }, + "provenance/experiment-provenance.json": { + "bytes": 5894, + "sha256": "490b8844864ba2d8116ff898ec7a010ca31f40e900863a317f4b452dcdf2d836" + }, + "provenance/full-audit.json": { + "bytes": 363, + "sha256": "2616ce8641587452aafaf88f730bd4a9546c5cd5fdd18097e3d2bf9e6c42dec6" + }, + "provenance/native-reference-validation.json": { + "bytes": 11624, + "sha256": "41289ff92629e67bc77f5e1790cc6e638254d0ab516fd42246fe520e228bb766" + }, + "provenance/selection.json": { + "bytes": 552, + "sha256": "87972e47a26951aed4e2ac350da3a72c2fc054e1e70649c16f06ff270d724e27" + }, + "provenance/teacher-runtime.json": { + "bytes": 2104, + "sha256": "d208c7bacc2974395585615eb5baaac0c8f4663fb34f5a8bb40dfa777e01921d" + }, + "provenance/template-filter.json": { + "bytes": 1576, + "sha256": "f0e55ba2ce0805e04ec3369e4a4a35bc9ff006d10367753b23502e34805635b4" + }, + "requirements.txt": { + "bytes": 188, + "sha256": "4d2963280263d14196c338b3a20d098d6d5853a70c410504ba717c8b81decaf1" + }, + "runtime/README.md": { + "bytes": 1775, + "sha256": "a4f8b516d2da87f9c6313ca258680e418572a1ce607ee8d7c847950159498204" + }, + "runtime/download-teacher.py": { + "bytes": 193, + "sha256": "0f6138e4d46bac757bccbc376cdcb5d88e85c6f5c259412c83db59b4c598afc6" + }, + "runtime/mac-environment.lock.txt": { + "bytes": 540, + "sha256": "a0196b5cf9b10aa8dfdc7311a534ae032c8647fbcc5c0bb23477097878f0a6f7" + }, + "runtime/requirements-mac.txt": { + "bytes": 188, + "sha256": "4d2963280263d14196c338b3a20d098d6d5853a70c410504ba717c8b81decaf1" + }, + "runtime/requirements-teacher-cuda.txt": { + "bytes": 72, + "sha256": "cac0f3bfd6d461da8415ace38468217839cf8185b9ae52a9e3247adca3f834e6" + }, + "runtime/requirements-training-cuda.txt": { + "bytes": 299, + "sha256": "7b0c3222210c260c9528a94828344ac59405acefdc8118bbf22b652d1dfa792d" + }, + "runtime/serve-teacher.sh": { + "bytes": 686, + "sha256": "21971a0062a8ede7df97fc6930003fbacb4011729bbfb8ea19b82cecee1daa2b" + }, + "split-audit.json": { + "bytes": 1423, + "sha256": "5d4275fb3fc9f13e1b9d8b6e6a4f90783db607f43a3d4f886a81b332dc3cdf50" + }, + "tinyquery/__init__.py": { + "bytes": 80, + "sha256": "83d8a6e71e625d98eef78ce40796f522c4be10aa149deab3bc2c1849b6414ebc" + }, + "tinyquery/audit_data.py": { + "bytes": 3526, + "sha256": "b7bf574f8f1ea4c34de3b80c0fc98fd98272161c13ce82dafe70c4bea1bd9948" + }, + "tinyquery/average_checkpoints.py": { + "bytes": 2239, + "sha256": "ff2fe4c30cf7b881e651a91457920a6133b3f72db0f66c3e809b6beb83bfcc99" + }, + "tinyquery/baseline.py": { + "bytes": 5342, + "sha256": "645c48aad3a492288881cd4f87fd554fa7e19b9c04eee80464f42324a0f8a0a8" + }, + "tinyquery/chat.py": { + "bytes": 3706, + "sha256": "6afc8a67312f5a909964465656107432a899eca5e59eab0af8bbc68bbb9c5e58" + }, + "tinyquery/checkpoint_eval.py": { + "bytes": 2441, + "sha256": "b09e552ec0d4af7c892d581798baaf0b6dcb4c379dd13e5c66b73edd1ce44f0d" + }, + "tinyquery/concrete.py": { + "bytes": 4755, + "sha256": "b4f122bac7f1798dbdae3d1cbef53905435c2f5e76ed0a7761182e7330212780" + }, + "tinyquery/curriculum_data.py": { + "bytes": 5910, + "sha256": "b5eddaa3c392b275a8c23a65fc8364bb0e48b2207776b0e14ed46e0af1abcdf8" + }, + "tinyquery/data.py": { + "bytes": 15996, + "sha256": "4f406ea28f33eea0dbff006269cf5f7f2128567be73d1a13b94c3116a3003928" + }, + "tinyquery/evaluate.py": { + "bytes": 9364, + "sha256": "7bebc27747fc6b3d3ce79471decb42b0dd200c7c7404c366b2c621dd76568204" + }, + "tinyquery/export.py": { + "bytes": 2871, + "sha256": "cb179b6eecab9f8f1788b5825910414a4e05a786064ea20d13921031b82d7698" + }, + "tinyquery/filter_templates.py": { + "bytes": 2130, + "sha256": "42adf094bd0d87b8457ec93e72e65e22684530b8a185468ef5a62e118e57221a" + }, + "tinyquery/grounding_data.py": { + "bytes": 5630, + "sha256": "0c936853adf0bb21dd3709d12622c4a2a5a18fba3fbfcd00f41a77a7cd7bb3fa" + }, + "tinyquery/layout_data.py": { + "bytes": 8159, + "sha256": "de5cb05aaf6846a37cd0803ff41846f20d968cb8ccf030f6906789a7be95429a" + }, + "tinyquery/manual_eval.py": { + "bytes": 5385, + "sha256": "7fb830a9d8780e43b3253a71a593296e6f6c8bb27d18f5b717e01ce851d52edf" + }, + "tinyquery/model.py": { + "bytes": 9287, + "sha256": "6d2f02f31228a010176719f91e113f7c16870c7d4f1cf1f63218d530caccdc43" + }, + "tinyquery/native_sql.py": { + "bytes": 5910, + "sha256": "0343c466f19b59c4d86bae9dcccf676d247b86bb268d72e38f49e385fc22fd71" + }, + "tinyquery/package_dataset.py": { + "bytes": 2998, + "sha256": "46d40adb4b810e31aa228f532a88790b501784e8f65e058f2c81805f7d9c32de" + }, + "tinyquery/prepare.py": { + "bytes": 3427, + "sha256": "28dbce0678889c787e30ae348019b8dd05c41a479f76b8f720c6cacce78b890f" + }, + "tinyquery/recipes.py": { + "bytes": 3534, + "sha256": "90cb3c82dbc7f50fc9e7a3ffbef8ff1ca4bf011e92389fa0a63e189a7eef0fdc" + }, + "tinyquery/requirements-inference.txt": { + "bytes": 188, + "sha256": "4d2963280263d14196c338b3a20d098d6d5853a70c410504ba717c8b81decaf1" + }, + "tinyquery/requirements.txt": { + "bytes": 299, + "sha256": "7b0c3222210c260c9528a94828344ac59405acefdc8118bbf22b652d1dfa792d" + }, + "tinyquery/rescore.py": { + "bytes": 1797, + "sha256": "01a451361650e10a0992819599da0054162e0c927438a856f086584ed2166ba6" + }, + "tinyquery/scope_data.py": { + "bytes": 3056, + "sha256": "9596541b871b33682d9a22ab28949aa8f6809d4d25b7fb3e494b033795b0b039" + }, + "tinyquery/teacher_bench.py": { + "bytes": 2235, + "sha256": "29b9943bc93060ad1cc5fbe91961b1219e87e0333f08d93d1af6ee4d64004635" + }, + "tinyquery/teacher_templates.py": { + "bytes": 4131, + "sha256": "9c81deae8851da2767fb160720ea16ea8969a7a9094079260bd55a17827a25e2" + }, + "tinyquery/test_adapter.py": { + "bytes": 3207, + "sha256": "78093a8b4339a2cb39dec4132749ac6de019a834af7c9b4bdc24929e6f8ebe4c" + }, + "tinyquery/test_model.py": { + "bytes": 1990, + "sha256": "c118a42cc1e590878aa6728279460e87f9f52e1ae9035c48915450e424661d9a" + }, + "tinyquery/train.py": { + "bytes": 11493, + "sha256": "4529ec9bd24158ed58557d4e1ecb2698ee171c36db6d344c8c67785795ac45f8" + }, + "tinyquery/verify_concrete.py": { + "bytes": 3874, + "sha256": "701d706c6b7de9409408b727b176d1b2b7e0dae6f263c4aa432fe93a7ed34126" + }, + "tokenization.json": { + "bytes": 992, + "sha256": "f300d44db96f49fdc7a8ee30f67b8aaa39524023d93a4c045333906c8ee0c5af" + }, + "tokenizer.json": { + "bytes": 264747, + "sha256": "6e7ce36ebb3345785916cabccaeb6fff24335b3e88f831894ed8ef263396ed1b" + } + } +} \ No newline at end of file diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1d93fbf2b0a55ebd8da328f73646c0c6ba0346f4 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9489b5cde69c11f64b3e7031182e8248aa83790f1ad4aeefbf99a4e5e152e692 +size 279483882 diff --git a/provenance/experiment-provenance.json b/provenance/experiment-provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..c87c31fb08ccf3370728734abf596040970375fd --- /dev/null +++ b/provenance/experiment-provenance.json @@ -0,0 +1,143 @@ +{ + "experiment_start_utc": "2026-09-10T10:57:23Z", + "hard_deadline_utc": "2026-09-10T14:57:23Z", + "student_initialization": "All weights originate from random initialization during this experiment. Copy-head parameters were added later; no pretrained student weights were used.", + "teacher_role": "Language templates, direct paraphrases, and round-trip verification; reference SQL semantics constructed programmatically.", + "curriculum": [ + { + "data": "v1", + "examples": 85777, + "tokens": 22739579, + "purpose": "Initial semantic and language learning" + }, + { + "data": "v2", + "examples": 152667, + "tokens": 43182146, + "purpose": "Random character identifiers and tool names" + }, + { + "data": "v3", + "examples": 218022, + "tokens": 62519864, + "purpose": "Training-vocabulary chunks and varied MCP naming styles" + }, + { + "data": "v4", + "examples": 283377, + "tokens": 81750566, + "purpose": "Identifiers without fixed prefixes; copies restricted to context/question", + "sample_weights": { + "latest_65355_variants": 3, + "other_rows": 1 + } + }, + { + "data": "v5", + "examples": 302893, + "tokens": 87335945, + "purpose": "Exact copying of project IDs and schema discovery table names", + "sample_weights": { + "scope_and_discovery_variants": 4, + "unprefixed_identifier_variants": 3, + "other_rows": 1 + }, + "validation_selection": "All 1,200 validation records, token-weighted loss" + }, + { + "data": "v6", + "examples": 320115, + "purpose": "New schema families with related table/parent/project identifiers; authored comparison contrasts; conservative training-template audit filtering", + "new_scenarios_before_filter": 6000, + "new_rows_before_filter": 24000, + "excluded_rows": 6778, + "validation_selection": "All 1,200 validation records, token-weighted response loss", + "tokens": 92098292, + "response_tokens": 12137844 + }, + { + "data": "v7", + "purpose": "Variable schema layouts, random qualified tool names, authored natural city requests and Hindi lexical mappings", + "new_layout_variants": 8826, + "new_runtime_name_variants": 13635, + "new_city_language_rows": 4800, + "new_city_scenario_families": 1200, + "new_variant_sample_weight": 6, + "development_observation": "An informal default-schema demo failed on a shorter schema and natural language. This stage addresses that development failure; the demo is not a held-out benchmark.", + "examples": 347376, + "tokens": 100224712, + "response_tokens": 13163249 + } + ], + "implementation_commits": [ + "f24d869", + "c1ce778", + "c9e8d77", + "f6f0b7b", + "780db6a", + "4875531", + "f713a08", + "69e19ff", + "1f798c7", + "47267b7", + "5d7a622", + "3ebf37a", + "eecb176", + "1c50583", + "364753b", + "a8f336f", + "1049d88", + "9729c63" + ], + "architecture_reference": "https://aclanthology.org/P17-1099/", + "research_claim": "Established decoder components and a learned pointer-generator adaptation. No claim of a new research architecture or controlled ablation.", + "template_audit": { + "teacher": "Qwen/Qwen3.8-27B-FP8", + "mtp": 3, + "jobs": 330, + "template_instances": 1313, + "accepted": 1276, + "flagged": 37, + "training_rows_excluded": 6778, + "scope": "Training SQL templates only; held-out validation/test/manual phrases excluded", + "caveat": "Earlier stages used pre-filter corpus; final continuation uses filtered data" + }, + "manual_split": "Additional hand-written phrasing templates rendered on 40 test scenario families; no shared train/validation scenarios. Manual and test are not independent schema-family samples.", + "development_split": "192 schema/tool-name perturbations of 140 validation families; no training, test or manual families. Used for development, not an independent test.", + "selection": { + "final_rule": "Among frozen evaluated candidates, maximize the mean of full validation task success and development task success; use validation response loss to select additional candidate snapshots. Final test and manual results are read only after weights are selected.", + "rationale": "Standard validation alone missed the schema-layout failure found in the development demo." + }, + "refinement": { + "data": "v7, unchanged", + "start_utc": "2026-09-10T13:40:00Z", + "planned_minutes": 18, + "batch": 128, + "learning_rate": 0.0001, + "prompt_weight": 0, + "response_weight": 1, + "action_auxiliary_weight": 0.05, + "rationale": "Focus late optimization on output actions after earlier language learning; reduce unpredictable-input LM gradients. Selection can retain an earlier checkpoint if refinement does not improve development scores." + }, + "selected_checkpoint": { + "step": 25715, + "sha256": "9489b5cde69c11f64b3e7031182e8248aa83790f1ad4aeefbf99a4e5e152e692", + "frozen_at_utc": "2026-09-10T14:00:36.513800+00:00", + "test_and_manual_excluded_from_selection": true + }, + "selected_checkpoint_counters": { + "step": 25715, + "processed_tokens": 595067301, + "response_tokens": 78367410, + "training_seconds": 5092.33953666687, + "random_initialization": true + }, + "completed_trainer_counters": { + "step": 32642, + "processed_tokens": 851198143, + "response_tokens": 113051294, + "training_seconds": 7099.562133073807, + "note": "Retained-lineage trainer counters include validation/checkpoint time, exclude discarded work/model loading. Last checkpoint was not selected." + }, + "release_archive_limitation": "Vast.ai SSH refused connections after final evaluation completed; local selected weights and datasets were already secured. Some raw remote logs and final optimizer state were not retrieved." +} \ No newline at end of file diff --git a/provenance/full-audit.json b/provenance/full-audit.json new file mode 100644 index 0000000000000000000000000000000000000000..ad30a4ff73f4bc58bc645b3cc25a2c62179f2088 --- /dev/null +++ b/provenance/full-audit.json @@ -0,0 +1,13 @@ +{ + "rows": 347376, + "unique_ids": 347376, + "unique_prompts": 347376, + "sql_rows": 262468, + "unique_sql_cases": 151920, + "distinct_context_sql_compilations": 160253, + "fixtures_per_sql_case": 2, + "sql_engine": "SQLite after dialect parsing/adaptation; native coverage reported separately", + "seconds": 97.74559593200684, + "errors": [], + "passed": true +} \ No newline at end of file diff --git a/provenance/native-reference-validation.json b/provenance/native-reference-validation.json new file mode 100644 index 0000000000000000000000000000000000000000..d2feeff5c9ff824c8cc06da8c66bd01d87793974 --- /dev/null +++ b/provenance/native-reference-validation.json @@ -0,0 +1,474 @@ +{ + "seconds": 0.32275891304016113, + "cases": 66, + "gold_executes": 66, + "prediction_equivalent": null, + "engine_versions": { + "mysql": "8.0.46", + "postgresql": "16.15" + }, + "results": [ + { + "id": "c4df2d6a92784c925f92_en", + "backend": "supabase", + "operation": "all", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "4996e2672571480e6c60_en", + "backend": "mysql", + "operation": "project", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "dbb8cd8445dd19a458e2_en", + "backend": "supabase", + "operation": "eq", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "39205adac55e98cd6749_en", + "backend": "supabase", + "operation": "project_eq", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "5fc77c83422c0e74572f_en", + "backend": "supabase", + "operation": "gt", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "4217b243f42c7af06169_en", + "backend": "mysql", + "operation": "lt", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "13ac9d307d6b101b8705_en", + "backend": "mysql", + "operation": "gte", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "467e53c90a42cd9935a3_en", + "backend": "mysql", + "operation": "lte", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "e0b7adca29f0eb32fe2c_en", + "backend": "mysql", + "operation": "between", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "1667f375dfed0ad934a4_en", + "backend": "mysql", + "operation": "and", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "328b77dd66cdf8cabfd8_en", + "backend": "mysql", + "operation": "or", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "5c1ca9682dbf9c4df2a1_en", + "backend": "supabase", + "operation": "null", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "7dfb47d110d38ff48e06_en", + "backend": "mysql", + "operation": "not_null", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "33d0ab3caa0fba0e3985_en", + "backend": "mysql", + "operation": "count", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "ad6bf3f875b067153767_en", + "backend": "supabase", + "operation": "count_eq", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "bc42ac714674eb7e9508_en", + "backend": "supabase", + "operation": "sum", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "5815b9336617abb93d33_en", + "backend": "supabase", + "operation": "avg", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "3487e6b7bf3881dd7e68_en", + "backend": "mysql", + "operation": "max", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "f39ec09a8da50e72bdd5_en", + "backend": "supabase", + "operation": "min", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "bc6f054aa3be632d1bce_en", + "backend": "mysql", + "operation": "distinct", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "dae8c9c830c197ee8e74_en", + "backend": "mysql", + "operation": "sort_desc", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "640719c646d2ad1c92e6_en", + "backend": "mysql", + "operation": "sort_asc", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "dc2e72d9a61704c6679c_en", + "backend": "mysql", + "operation": "top", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "9b0921e93df3d40ae856_en", + "backend": "supabase", + "operation": "bottom", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "c23d51b5c96bb4b300bf_en", + "backend": "supabase", + "operation": "group_count", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "dafbcd52face40a7b678_en", + "backend": "mysql", + "operation": "group_sum", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "f551f3aa4afc6f91a2d5_en", + "backend": "supabase", + "operation": "having", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "f9c2698db9e7dbe6ec8e_en", + "backend": "supabase", + "operation": "date_after", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "6016d266fd2535bc4479_en", + "backend": "supabase", + "operation": "year", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "ccbdc5b0dcc9fcc00608_en", + "backend": "supabase", + "operation": "month", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "068560b98b4de1eee24e_en", + "backend": "supabase", + "operation": "contains", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "7cf5c995d18de3fe3679_en", + "backend": "mysql", + "operation": "join", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "f0a44c49f89e568f5a97_en", + "backend": "supabase", + "operation": "join_filter", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "2d72fda52b91ee1560ab_en", + "backend": "mysql", + "operation": "all", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "029cd7c058965ed14b05_en", + "backend": "mysql", + "operation": "eq", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "327bced9ccce8adafb67_en", + "backend": "supabase", + "operation": "lt", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "c5a56e24a5878c81791a_en", + "backend": "supabase", + "operation": "gte", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "9925b01d3b059dd22a13_en", + "backend": "supabase", + "operation": "or", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "5ec7a0dca8e2804dbca9_en", + "backend": "supabase", + "operation": "not_null", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "8488c3423e53e10c7853_en", + "backend": "supabase", + "operation": "count", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "7839648878bf9feda17b_en", + "backend": "mysql", + "operation": "count_eq", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "e956931ef5531d79954c_en", + "backend": "mysql", + "operation": "avg", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "d2ac91503b4c4b8c4bbf_en", + "backend": "supabase", + "operation": "max", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "e27bab397ec7daf2a101_en", + "backend": "supabase", + "operation": "distinct", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "9912a7207dd6e403b2d5_en", + "backend": "supabase", + "operation": "top", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "90eda6200c21cebc9077_en", + "backend": "mysql", + "operation": "group_count", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "fe5044da918260a617a9_en", + "backend": "supabase", + "operation": "group_sum", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "2daba35926021b3697eb_en", + "backend": "mysql", + "operation": "date_after", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "9992776e0fbabf4da80e_en", + "backend": "mysql", + "operation": "month", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "b654c9f6e4c53883151c_en", + "backend": "supabase", + "operation": "join", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "3cfa2682031af21eedd8_en", + "backend": "supabase", + "operation": "project", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "403211bcb94202245467_en", + "backend": "mysql", + "operation": "project_eq", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "9ef4d4f5d5bcf5858087_en", + "backend": "supabase", + "operation": "and", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "8c437f27ac8f48915f9a_en", + "backend": "mysql", + "operation": "min", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "4c485cb9781970855140_en", + "backend": "supabase", + "operation": "sort_desc", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "0bdc9f428b0896d7e38a_en", + "backend": "supabase", + "operation": "sort_asc", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "c4a83c9634139691ddde_en", + "backend": "mysql", + "operation": "having", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "3b0a2346d3ae970242c1_en", + "backend": "mysql", + "operation": "year", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "cd68dcf59fb97c84247d_en", + "backend": "mysql", + "operation": "contains", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "6569be8d680d87bb283d_en", + "backend": "mysql", + "operation": "gt", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "69aa7071e087609fac06_en", + "backend": "supabase", + "operation": "lte", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "49dc5daf9217fe08b1c4_en", + "backend": "mysql", + "operation": "sum", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "f637ccd41642ab190d04_en", + "backend": "mysql", + "operation": "null", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "0aec7748ac00fd2195f1_en", + "backend": "mysql", + "operation": "join_filter", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "56594767710f98b9b347_en", + "backend": "supabase", + "operation": "between", + "gold_executes": true, + "prediction_equivalent": null + }, + { + "id": "eb4c2f1cc33712f0b82e_en", + "backend": "mysql", + "operation": "bottom", + "gold_executes": true, + "prediction_equivalent": null + } + ] +} \ No newline at end of file diff --git a/provenance/selection.json b/provenance/selection.json new file mode 100644 index 0000000000000000000000000000000000000000..d6d5b71e043304e720101de7fd321ab3f59771a1 --- /dev/null +++ b/provenance/selection.json @@ -0,0 +1,19 @@ +{ + "checkpoint": "/workspace/tinyquery/candidates/step-25715/model.safetensors", + "step": 25715, + "checkpoint_sha256": "9489b5cde69c11f64b3e7031182e8248aa83790f1ad4aeefbf99a4e5e152e692", + "development_scores": { + "validation": { + "correct": 1174, + "total": 1200, + "rate": 0.9783333333333334 + }, + "development": { + "correct": 188, + "total": 192, + "rate": 0.9791666666666666 + } + }, + "selection_score": 0.97875, + "rule": "Equal mean of full validation and development task success. Test/manual excluded." +} \ No newline at end of file diff --git a/provenance/teacher-runtime.json b/provenance/teacher-runtime.json new file mode 100644 index 0000000000000000000000000000000000000000..96253046214806ac88f1e82437e0a72cdc9f9688 --- /dev/null +++ b/provenance/teacher-runtime.json @@ -0,0 +1,81 @@ +{ + "teacher": "Qwen/Qwen3.8-27B-FP8", + "revision": "017b9c7af6b5689d5dd426a76e0bc077eb5ca20a", + "server": { + "vllm": "0.28.0", + "hardware": "RTX PRO 6000 Blackwell Max-Q Workstation Edition, 96 GB", + "context": 8192, + "max_num_seqs": 32, + "gpu_memory_utilization": 0.85, + "language_model_only": true, + "fp8_backend": "CUTLASS" + }, + "phases": [ + { + "name": "language_templates", + "requests": 45, + "concurrency": 16, + "temperature": 0.8, + "top_p": 0.95, + "top_k": 20, + "thinking": false, + "mtp": false, + "max_tokens": 4096 + }, + { + "name": "direct_paraphrases", + "requests": 1500, + "concurrency": 32, + "temperature": 0.7, + "top_p": 0.95, + "top_k": 20, + "thinking": false, + "mtp": false, + "max_tokens": 500 + }, + { + "name": "round_trip_verification", + "requests": 1500, + "temperature": 0, + "thinking": false, + "mtp": true, + "num_speculative_tokens": 3, + "max_tokens": 700 + }, + { + "name": "training_template_semantic_audit", + "requests": 330, + "template_instances": 1313, + "concurrency": 32, + "temperature": 0, + "thinking": false, + "mtp": true, + "num_speculative_tokens": 3, + "max_tokens": 700, + "accepted_instances": 1276, + "flagged_instances": 37 + } + ], + "matched_benchmark": { + "concurrency": 32, + "mtp_off_tps": [ + 688.80098, + 693.58296 + ], + "mtp_3_tps": [ + 926.42637, + 935.17359 + ], + "warmup": "one full request round per configuration before measured rounds", + "measured_rounds_per_configuration": 2, + "requests_per_round": 64, + "valid_json_per_configuration": 128, + "truncated_per_configuration": 0, + "temperature": 0.7, + "top_p": 0.8, + "top_k": 20, + "presence_penalty": 0, + "repetition_penalty": 1.0 + }, + "scope": "MTP speeds inference; it does not certify semantic correctness. Initial template/paraphrase generation preceded MTP; verification used MTP. No claim of a globally optimal configuration." +} \ No newline at end of file diff --git a/provenance/template-filter.json b/provenance/template-filter.json new file mode 100644 index 0000000000000000000000000000000000000000..dc912f49ccbffd62cb98c817f0f86ad763c25fb1 --- /dev/null +++ b/provenance/template-filter.json @@ -0,0 +1,48 @@ +{ + "audit_jobs": 330, + "accepted_template_instances": 1276, + "flagged_template_variants": 37, + "excluded_training_rows": 6778, + "remaining_training_rows": 320115, + "exclusions": { + "bottom/hi/2": 215, + "contains/noisy_en/9": 196, + "contains/hi/4": 145, + "contains/hinglish/7": 195, + "bottom/noisy_en/2": 208, + "contains/en/9": 185, + "contains/hi/7": 168, + "contains/hinglish/8": 202, + "lt/hinglish/4": 144, + "not_null/hi/5": 196, + "top/en/3": 216, + "bottom/en/2": 179, + "contains/noisy_en/1": 164, + "contains/hi/2": 221, + "lt/en/4": 190, + "not_null/hinglish/5": 164, + "contains/en/1": 200, + "contains/hi/9": 219, + "contains/hinglish/2": 168, + "contains/en/7": 187, + "contains/hinglish/9": 180, + "bottom/hinglish/2": 172, + "contains/en/2": 213, + "contains/noisy_en/8": 217, + "contains/hi/1": 147, + "contains/en/4": 219, + "contains/noisy_en/2": 176, + "max/en/9": 151, + "lt/en/5": 161, + "contains/hi/8": 175, + "not_null/en/5": 172, + "contains/noisy_en/4": 228, + "contains/en/8": 122, + "contains/hinglish/1": 168, + "contains/noisy_en/7": 189, + "contains/hinglish/4": 189, + "lt/hi/4": 137 + }, + "policy": "Conservative exclusion after SQL round-trip disagreement; disagreement alone does not prove the teacher was correct. Validation, test, manual, direct verified paraphrases and separately authored contrasts remain unchanged.", + "limitation": "Earlier curriculum stages trained on these rows before the audit; this filtering does not undo earlier exposure." +} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..577c995eca8e5a9b85f641b58916510e0aa53d2f --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +# Tested with PyTorch 2.13.0 on CUDA and 2.14.0 on macOS. +torch>=2.13,<3 +numpy>=2.2,<3 +tokenizers==0.22.2 +safetensors>=0.8,<1 +sqlglot==30.18.0 +jsonschema>=4.26,<5 +huggingface-hub>=1.28,<2 diff --git a/runtime/README.md b/runtime/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b968dd281c656c93b39b281b120ba59191a5b25f --- /dev/null +++ b/runtime/README.md @@ -0,0 +1,16 @@ +# Runtime and reproducibility + +The student runs locally on Apple Silicon MPS, CUDA, or CPU. Use Python 3.12 and install `requirements-mac.txt` (or the root `requirements.txt`). `mac-environment.lock.txt` records the actual local environment. CUDA training and teacher requirements belong in separate GPU environments; vLLM FP8 teacher serving is not a Mac runtime. + +`serve-teacher.sh` reconstructs the recorded working launch settings. The exact downloaded teacher revision is pinned in `download-teacher.py`. CUDA training used PyTorch 2.13.0+cu130; use the appropriate official CUDA wheel index. The teacher used vLLM 0.28.0 and Transformers 5.15.1. Do not install the CUDA environment into the Mac inference environment. + +The published dataset freezes the final corpus; original templates, prompts, verification records and programmatic generators are included in the companion dataset and source package. Training evolved through seven curricula; a fresh run on the final corpus is a new experiment, not a bitwise replay. Earlier training encountered 6,778 rows later filtered out. The released weights are selected by validation and development success, not the last trainer checkpoint. The final optimizer state and some raw remote logs could not be retrieved after the rented SSH endpoint refused connections. + +Example fresh training from the published final corpus: + +```sh +python -m tinyquery.prepare --data path/to/extracted-jsonl-splits +python -m tinyquery.train --data path/to/extracted-jsonl-splits --out runs/new --copy-dim 128 --minutes 120 +``` + +For the exact dataset tokenizer, pass the existing tokenizer through the prepare script's documented reuse option (`python -m tinyquery.prepare --help`). A new randomly initialized run can obtain different results. diff --git a/runtime/download-teacher.py b/runtime/download-teacher.py new file mode 100644 index 0000000000000000000000000000000000000000..e9cbe21cda4a6c77e57ba09dc4cc159494ca15cd --- /dev/null +++ b/runtime/download-teacher.py @@ -0,0 +1,2 @@ +from huggingface_hub import snapshot_download +snapshot_download("Qwen/Qwen3.8-27B-FP8", revision="017b9c7af6b5689d5dd426a76e0bc077eb5ca20a", local_dir="teacher/Qwen3.8-27B-FP8", max_workers=8) diff --git a/runtime/mac-environment.lock.txt b/runtime/mac-environment.lock.txt new file mode 100644 index 0000000000000000000000000000000000000000..fad242d2822d797acf9567e269706b37eba290ac --- /dev/null +++ b/runtime/mac-environment.lock.txt @@ -0,0 +1,32 @@ +anyio==4.15.1 +attrs==26.1.0 +certifi==2026.7.22 +click==8.5.0 +filelock==3.32.6 +fsspec==2026.7.0 +h11==0.16.0 +hf-xet==1.6.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface-hub==1.30.0 +idna==3.19 +jinja2==3.1.6 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +markupsafe==3.0.3 +mpmath==1.3.0 +networkx==3.6.1 +numpy==2.5.3 +packaging==26.3 +pyarrow==25.0.1 +pyyaml==6.0.3 +referencing==0.37.0 +rpds-py==2026.6.3 +safetensors==0.8.0 +setuptools==84.0.0 +sqlglot==30.18.0 +sympy==1.14.0 +tokenizers==0.23.2 +torch==2.14.0 +tqdm==4.70.0 +typing-extensions==4.16.0 diff --git a/runtime/requirements-mac.txt b/runtime/requirements-mac.txt new file mode 100644 index 0000000000000000000000000000000000000000..577c995eca8e5a9b85f641b58916510e0aa53d2f --- /dev/null +++ b/runtime/requirements-mac.txt @@ -0,0 +1,8 @@ +# Tested with PyTorch 2.13.0 on CUDA and 2.14.0 on macOS. +torch>=2.13,<3 +numpy>=2.2,<3 +tokenizers==0.22.2 +safetensors>=0.8,<1 +sqlglot==30.18.0 +jsonschema>=4.26,<5 +huggingface-hub>=1.28,<2 diff --git a/runtime/requirements-teacher-cuda.txt b/runtime/requirements-teacher-cuda.txt new file mode 100644 index 0000000000000000000000000000000000000000..6d779ad77d9ad6fe451d81d436c29c03fc64716c --- /dev/null +++ b/runtime/requirements-teacher-cuda.txt @@ -0,0 +1,4 @@ +vllm==0.28.0 +torch==2.13.0 +transformers==5.15.1 +huggingface-hub==1.28.0 diff --git a/runtime/requirements-training-cuda.txt b/runtime/requirements-training-cuda.txt new file mode 100644 index 0000000000000000000000000000000000000000..2f1592990286f98a6874434f25aa79b54109f313 --- /dev/null +++ b/runtime/requirements-training-cuda.txt @@ -0,0 +1,11 @@ +# Training environment used on the rented CUDA instance; torch includes its CUDA runtime. +torch==2.13.0 +numpy==2.2.6 +tokenizers==0.22.2 +safetensors==0.8.0 +sqlglot==30.18.0 +jsonschema==4.26.0 +huggingface-hub==1.28.0 +# Only needed for native database evaluation: +psycopg[binary]==3.3.5 +pymysql==1.2.0 diff --git a/runtime/serve-teacher.sh b/runtime/serve-teacher.sh new file mode 100644 index 0000000000000000000000000000000000000000..73813856b0efce3adf3398fd1967899f789c3ce3 --- /dev/null +++ b/runtime/serve-teacher.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail +# Reconstructed from recorded working settings; requires the rented CUDA GPU. +# Supply a local pinned snapshot path, or download the recorded revision first. +: "${TEACHER_MODEL_PATH:?Set TEACHER_MODEL_PATH to the downloaded Qwen snapshot}" +exec vllm serve "$TEACHER_MODEL_PATH" \ + --served-model-name Qwen/Qwen3.8-27B-FP8 \ + --host 127.0.0.1 --port 18000 --tensor-parallel-size 1 \ + --language-model-only --max-model-len 8192 --max-num-seqs 32 \ + --gpu-memory-utilization 0.85 --reasoning-parser qwen3 \ + --compilation-config '{"cudagraph_capture_sizes":[1,2,4,8,16,32]}' \ + --speculative-config '{"method":"mtp","num_speculative_tokens":3}' diff --git a/split-audit.json b/split-audit.json new file mode 100644 index 0000000000000000000000000000000000000000..a1747f5b83e0b9f6172bdadac9cf5b47c19566d0 --- /dev/null +++ b/split-audit.json @@ -0,0 +1,76 @@ +{ + "splits": { + "train": { + "rows": 347376, + "unique_prompts": 347376, + "scenario_groups": 25145, + "unique_ids": 347376 + }, + "validation": { + "rows": 1200, + "unique_prompts": 1200, + "scenario_groups": 300, + "unique_ids": 1200 + }, + "test": { + "rows": 1196, + "unique_prompts": 1196, + "scenario_groups": 299, + "unique_ids": 1196 + }, + "manual": { + "rows": 160, + "unique_prompts": 160, + "scenario_groups": 40, + "unique_ids": 160 + }, + "development": { + "rows": 192, + "unique_prompts": 192, + "scenario_groups": 140, + "unique_ids": 192 + } + }, + "overlap": { + "train-validation": { + "prompts": 0, + "scenarios": 0 + }, + "train-test": { + "prompts": 0, + "scenarios": 0 + }, + "train-manual": { + "prompts": 0, + "scenarios": 0 + }, + "train-development": { + "prompts": 0, + "scenarios": 0 + }, + "validation-test": { + "prompts": 0, + "scenarios": 0 + }, + "validation-manual": { + "prompts": 0, + "scenarios": 0 + }, + "validation-development": { + "prompts": 0, + "scenarios": 140 + }, + "test-manual": { + "prompts": 0, + "scenarios": 40 + }, + "test-development": { + "prompts": 0, + "scenarios": 0 + }, + "manual-development": { + "prompts": 0, + "scenarios": 0 + } + } +} \ No newline at end of file diff --git a/tinyquery/__init__.py b/tinyquery/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..11ff1acc0ce66dfa8eb70ab4edb69d3aba2de0b2 --- /dev/null +++ b/tinyquery/__init__.py @@ -0,0 +1 @@ +"""TinyQuery: an explicitly bounded tool-calling model trained from scratch.""" diff --git a/tinyquery/__pycache__/__init__.cpython-312.pyc b/tinyquery/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1125aa8e8ad63ae99633e6ff43db417e50ce8f38 Binary files /dev/null and b/tinyquery/__pycache__/__init__.cpython-312.pyc differ diff --git a/tinyquery/__pycache__/chat.cpython-312.pyc b/tinyquery/__pycache__/chat.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..de00d1513eb9f4a24825d1a5d7e828b4ec40bf77 Binary files /dev/null and b/tinyquery/__pycache__/chat.cpython-312.pyc differ diff --git a/tinyquery/__pycache__/data.cpython-312.pyc b/tinyquery/__pycache__/data.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..82de171fd679158f38aa1155e27557bfe7b33454 Binary files /dev/null and b/tinyquery/__pycache__/data.cpython-312.pyc differ diff --git a/tinyquery/__pycache__/evaluate.cpython-312.pyc b/tinyquery/__pycache__/evaluate.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a44e92998a2cfc559325f3dae80776ac21dadbba Binary files /dev/null and b/tinyquery/__pycache__/evaluate.cpython-312.pyc differ diff --git a/tinyquery/__pycache__/model.cpython-312.pyc b/tinyquery/__pycache__/model.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3fbc888dfcf7a0d3bb5c733850387280fd9d660d Binary files /dev/null and b/tinyquery/__pycache__/model.cpython-312.pyc differ diff --git a/tinyquery/__pycache__/recipes.cpython-312.pyc b/tinyquery/__pycache__/recipes.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bf6dadf286bbd39aac48ee9700df83537c1ab86a Binary files /dev/null and b/tinyquery/__pycache__/recipes.cpython-312.pyc differ diff --git a/tinyquery/audit_data.py b/tinyquery/audit_data.py new file mode 100644 index 0000000000000000000000000000000000000000..71f5b2e0dae2ac16d2eb5b5b9308610b8f9f0b89 --- /dev/null +++ b/tinyquery/audit_data.py @@ -0,0 +1,59 @@ +"""Audit every reference action and execute each distinct SQL case on two fixtures.""" +import argparse +import hashlib +import json +import sqlite3 +from pathlib import Path +import time +from jsonschema import Draft202012Validator +from tinyquery.data import SQL_OPS,compact,serialize,validate_sql,sqlite_sql +from tinyquery.evaluate import check_action + + +def main(): + p=argparse.ArgumentParser();p.add_argument('--data',required=True);p.add_argument('--out',required=True) + args=p.parse_args();start=time.time();ids=set();prompts=set();queries=set();context_queries=set();validators={};errors=[];count=0;sql_rows=0 + for line in Path(args.data).open(): + row=json.loads(line);count+=1 + try: + assert row['id'] not in ids,'Duplicate ID' + ids.add(row['id']);fingerprint=hashlib.sha256(row['prompt'].encode()).hexdigest() + assert fingerprint not in prompts,'Duplicate prompt' + prompts.add(fingerprint) + assert row['prompt']==serialize(row['context'],row['question']),'Prompt serialization differs' + assert row['response']==compact(row['target']),'Response serialization differs' + action=row['target'] + if action['action']=='call': + assert set(action)=={'action','name','arguments'} + tool=next(t for t in row['context']['tools'] if t['name']==action['name']) + schema=compact(tool['inputSchema']) + if schema not in validators:validators[schema]=Draft202012Validator(tool['inputSchema']) + validators[schema].validate(action['arguments']) + else:check_action(action,row['context']) + if row['operation'] in SQL_OPS: + sql_rows+=1;a=action['arguments'];sql=a.get('sql',a.get('query')) + context_key=hashlib.sha256(compact([row['backend'],row['context']['schema'],sql]).encode()).hexdigest() + if context_key not in context_queries: + import sqlglot + db=sqlite3.connect(':memory:');db.create_function('YEAR',1,lambda x:0);db.create_function('MONTH',1,lambda x:0) + try: + for ddl in row['context']['schema']:db.execute(ddl) + db.execute(sqlite_sql(sqlglot.parse_one(sql,read='postgres' if row['backend']=='supabase' else 'mysql'))) + finally:db.close() + context_queries.add(context_key) + key=hashlib.sha256(compact([row['backend'],row['slots'],sql]).encode()).hexdigest() + if key not in queries: + validate_sql(sql,row['backend'],row['slots']);queries.add(key) + except Exception as exc: + errors.append({'id':row['id'],'error':type(exc).__name__+': '+str(exc)}) + if len(errors)>=20:break + if count%10000==0:print(json.dumps({'rows':count,'unique_sql':len(queries),'seconds':time.time()-start}),flush=True) + report={'rows':count,'unique_ids':len(ids),'unique_prompts':len(prompts),'sql_rows':sql_rows,'unique_sql_cases':len(queries), + 'distinct_context_sql_compilations':len(context_queries), + 'fixtures_per_sql_case':2,'sql_engine':'SQLite after dialect parsing/adaptation; native coverage reported separately', + 'seconds':time.time()-start,'errors':errors,'passed':not errors} + Path(args.out).write_text(json.dumps(report,indent=2));print(json.dumps(report),flush=True) + if errors:raise SystemExit(1) + + +if __name__=='__main__':main() diff --git a/tinyquery/average_checkpoints.py b/tinyquery/average_checkpoints.py new file mode 100644 index 0000000000000000000000000000000000000000..2d553aa6e1be9b4812f9274a399e7d1382f15e70 --- /dev/null +++ b/tinyquery/average_checkpoints.py @@ -0,0 +1,33 @@ +"""Average compatible checkpoints from the same randomly initialized training lineage.""" +import argparse +import json +from pathlib import Path +import torch +from safetensors.torch import load_file,save_file +from tinyquery.prepare import file_sha256 + + +def main(): + p=argparse.ArgumentParser();p.add_argument('--checkpoints',nargs='+',required=True);p.add_argument('--weights',nargs='+',type=float) + p.add_argument('--out',required=True);args=p.parse_args();weights=args.weights or [1]*len(args.checkpoints) + assert len(weights)==len(args.checkpoints) and all(w>=0 for w in weights) and sum(weights)>0 + weights=[w/sum(weights) for w in weights];total={};config=None;sources=[] + for name,weight in zip(args.checkpoints,weights): + path=Path(name);c=json.loads((path.parent/'config.json').read_text());assert config is None or config==c;config=c + info=json.loads((path.parent/'checkpoint-info.json').read_text());assert info['random_initialization'] + state=load_file(str(path));assert not total or total.keys()==state.keys() + for key,value in state.items(): + if key not in total:total[key]=value.float()*weight + else:total[key].add_(value.float(),alpha=weight) + sources.append({'checkpoint':str(path),'sha256':file_sha256(path),'weight':weight,'info':info});del state + dest=Path(args.out);dest.mkdir(parents=True,exist_ok=True) + save_file({k:v.to(torch.bfloat16).contiguous() for k,v in total.items()},str(dest/'model.safetensors'), + metadata={'method':'weighted_parameter_average','random_initialization':'true'}) + (dest/'config.json').write_text(json.dumps(config,indent=2)) + info={'method':'weighted_parameter_average','sources':sources,'random_initialization':True, + 'step':max(s['info']['step'] for s in sources),'step_interpretation':'Latest source step; these are averaged weights, not that raw checkpoint.'} + for key in ['processed_tokens','response_tokens','training_seconds']:info[key]=max(s['info'][key] for s in sources) + (dest/'checkpoint-info.json').write_text(json.dumps(info,indent=2));print(json.dumps({'out':str(dest),'sources':[(s['info']['step'],s['weight']) for s in sources]})) + + +if __name__=='__main__':main() diff --git a/tinyquery/baseline.py b/tinyquery/baseline.py new file mode 100644 index 0000000000000000000000000000000000000000..e00967a4e9ead3be02232cbdaf92c73525b966bb --- /dev/null +++ b/tinyquery/baseline.py @@ -0,0 +1,90 @@ +"""Transparent TF-IDF nearest-question baseline; copies the retrieved action unchanged.""" +import argparse +from collections import Counter,defaultdict +import json +import math +from pathlib import Path +import re +import time +import numpy as np +from tinyquery.evaluate import score,aggregate + + +def bind_context(text,source,destination): + """Bind by tool descriptions and DDL positions, using no expected answer or slots.""" + import sqlglot + from sqlglot import exp + action=json.loads(text) + if action.get('action')!='call':return text + old=next(t for t in source['tools'] if t['name']==action['name']) + normalize=lambda s:s.replace('PostgreSQL','SQL').replace('MySQL','SQL') + candidates=[t for t in destination['tools'] if normalize(t['description'])==normalize(old['description'])] + if not candidates:return text + tool=candidates[0];props=tool['inputSchema']['properties'];args=dict(action['arguments']) + sqlkey=next((k for k in ['sql','query'] if isinstance(args.get(k),str) and args[k].startswith('SELECT ')),None) + if sqlkey: + targetkey=next((k for k in ['sql','query'] if k in props),sqlkey) + tables={};columns={} + for old_ddl,new_ddl in zip(source['schema'],destination['schema']): + a=sqlglot.parse_one(old_ddl).this;b=sqlglot.parse_one(new_ddl).this + tables[a.this.name]=b.this.name + for c,d in zip(a.expressions,b.expressions): + if isinstance(c,exp.ColumnDef) and isinstance(d,exp.ColumnDef):columns[c.name]=d.name + query=sqlglot.parse_one(args.pop(sqlkey),read='postgres' if source['backend']=='supabase' else 'mysql') + def rename(node): + if isinstance(node,exp.Table) and node.name in tables:node.set('this',exp.to_identifier(tables[node.name])) + if isinstance(node,exp.Column): + if node.name in columns:node.set('this',exp.to_identifier(columns[node.name])) + if node.table in tables:node.set('table',exp.to_identifier(tables[node.table])) + return node + args[targetkey]=query.transform(rename).sql(dialect='postgres' if destination['backend']=='supabase' else 'mysql')+';' + args={k:v for k,v in args.items() if k in props} + if 'project_id' in props:args['project_id']=destination['project_id'] + return json.dumps({'action':'call','name':tool['name'],'arguments':args},ensure_ascii=False,separators=(',',':')) + + +def words(text): + return re.findall(r'[^\W_]+|_',text.lower(),re.UNICODE) + + +def main(): + p=argparse.ArgumentParser(); p.add_argument('--train',required=True); p.add_argument('--data',required=True) + p.add_argument('--out',required=True);p.add_argument('--bind-context',action='store_true') + args=p.parse_args(); start=time.time() + training=[]; seen=set(); counts=[]; df=Counter() + for line in Path(args.train).open(): + row=json.loads(line) + if row['question'] in seen: continue + seen.add(row['question']); terms=Counter(words(row['question'])) + training.append((row['id'],row['response'],row['context'] if args.bind_context else None)); counts.append(terms); df.update(terms.keys()) + n=len(training); idf={word:math.log((1+n)/(1+freq))+1 for word,freq in df.items()} + postings=defaultdict(list) + for i,terms in enumerate(counts): + weighted={word:(1+math.log(freq))*idf[word] for word,freq in terms.items()} + norm=math.sqrt(sum(v*v for v in weighted.values())) or 1 + for word,value in weighted.items(): postings[word].append((i,value/norm)) + postings={word:(np.array([i for i,_ in pairs]),np.array([v for _,v in pairs],dtype=np.float32)) + for word,pairs in postings.items()} + results=[]; groups=defaultdict(list); out=Path(args.out); out.parent.mkdir(parents=True,exist_ok=True) + with out.open('w') as stream: + for line in Path(args.data).open(): + row=json.loads(line); similarities=np.zeros(n,dtype=np.float32) + for word,freq in Counter(words(row['question'])).items(): + if word in postings: + indices,weights=postings[word]; similarities[indices]+=(1+math.log(freq))*idf[word]*weights + index=int(similarities.argmax()); source,text,context=training[index] + if args.bind_context: + try:text=bind_context(text,context,row['context']) + except (ValueError,KeyError,StopIteration,AttributeError):pass + metrics=score(row,text) + results.append(metrics) + for field in ['language','backend','operation']: groups[field+':'+row[field]].append(metrics) + stream.write(json.dumps({'id':row['id'],'retrieved_id':source,'output':text,'metrics':metrics},ensure_ascii=False)+'\n') + report={'baseline':'TF-IDF nearest training question; copy action unchanged, no tool/schema adaptation', + 'training_questions':n,'examples':len(results),'seconds':time.time()-start,'metrics':aggregate(results), + 'groups':{key:aggregate(value) for key,value in groups.items()}} + if args.bind_context:report['baseline']='TF-IDF nearest question plus tool-description and DDL-position binding; no expected answer or gold slots used' + out.with_suffix('.summary.json').write_text(json.dumps(report,indent=2)); print(json.dumps(report['metrics'])) + + +if __name__=='__main__': main() diff --git a/tinyquery/chat.py b/tinyquery/chat.py new file mode 100644 index 0000000000000000000000000000000000000000..906aad9853c803efb5179a688edb05afbb4ba914 --- /dev/null +++ b/tinyquery/chat.py @@ -0,0 +1,68 @@ +"""Stream the student's raw JSON response from a checkpoint; optionally render an MCP request.""" +import argparse +import json +import sys +import time +from pathlib import Path +import torch +from tokenizers import Tokenizer,decoders +from tinyquery.data import serialize,make_tools +from tinyquery.evaluate import load_model,check_action,mcp_request,parse_action + + +def default_context(backend): + import random + tools,_,_,_=make_tools(backend,random.Random(4),'train','demo') + return {'backend':backend,'project_id':'demo', + 'schema':['CREATE TABLE customers (id INTEGER PRIMARY KEY, name TEXT, city TEXT, amount REAL);'], + 'tools':tools,'policy':'Read-only database access. Use provided tools. Ask when required information is missing.'} + + +def stream_response(model,tokenizer,prompt,max_tokens=160,stats=None): + ids=tokenizer.encode(prompt).ids + if len(ids)+max_tokens>model.config.context: raise ValueError('Prompt plus output budget exceeds model context') + device=next(model.parameters()).device; tokens=torch.tensor([ids],device=device) + past=None; decoder=decoders.DecodeStream(skip_special_tokens=True) + generated=[]; emitted='';start=time.perf_counter();first_token=None + with torch.inference_mode(): + for _ in range(max_tokens): + logits,past,_=model(tokens,past=past,use_cache=True,last_only=True) + token=int(logits[0,-1].argmax()); generated.append(token) + if first_token is None:first_token=time.perf_counter()-start + text=decoder.step(tokenizer,token) + if text: emitted+=text; yield text + if token==tokenizer.token_to_id('<|end|>'): break + tokens=torch.tensor([[token]],device=device) + full=tokenizer.decode(generated,skip_special_tokens=True) + if full.startswith(emitted) and len(full)>len(emitted): yield full[len(emitted):] + if stats is not None: + seconds=time.perf_counter()-start + stats.update(generated_tokens=len(generated),seconds=seconds,tokens_per_second=len(generated)/seconds, + first_token_seconds=first_token,device=str(device),includes_model_loading=False) + + +def main(): + p=argparse.ArgumentParser(); p.add_argument('question'); p.add_argument('--checkpoint',required=True) + p.add_argument('--tokenizer'); p.add_argument('--context',help='JSON file with backend, schema and MCP-style tools') + p.add_argument('--backend',choices=['mysql','supabase'],default='supabase') + p.add_argument('--tokens',type=int,default=160); p.add_argument('--mcp',action='store_true') + p.add_argument('--stats',action='store_true',help='Print measured generation speed to stderr after streaming') + args=p.parse_args(); device='cuda' if torch.cuda.is_available() else ('mps' if torch.backends.mps.is_available() else 'cpu') + torch.set_num_threads(4) + tokenizer=Tokenizer.from_file(args.tokenizer or str(Path(args.checkpoint).parent/'tokenizer.json')) + model=load_model(args.checkpoint,device) + context=json.loads(Path(args.context).read_text()) if args.context else default_context(args.backend) + answer='';stats={} + for text in stream_response(model,tokenizer,serialize(context,args.question),args.tokens,stats): + print(text,end='',flush=True); answer+=text + print(flush=True) + if args.stats:print(json.dumps(stats),file=sys.stderr) + try: + action=check_action(parse_action(answer),context) + if args.mcp and action['action']=='call': print(json.dumps(mcp_request(action,context),ensure_ascii=False,indent=2)) + except Exception as exc: + print('Output validation failed:',str(exc),file=sys.stderr) + raise SystemExit(1) + + +if __name__=='__main__': main() diff --git a/tinyquery/checkpoint_eval.py b/tinyquery/checkpoint_eval.py new file mode 100644 index 0000000000000000000000000000000000000000..f727a1a6792cd0dcebb76b35ffb2a7f4f755cc26 --- /dev/null +++ b/tinyquery/checkpoint_eval.py @@ -0,0 +1,38 @@ +"""Freeze an owned training checkpoint and evaluate only development splits.""" +import argparse +import json +import os +from pathlib import Path +import subprocess +import sys +import torch +from safetensors.torch import save_file + + +def main(): + p=argparse.ArgumentParser();p.add_argument('--checkpoint',required=True);p.add_argument('--data',required=True) + p.add_argument('--out',required=True);args=p.parse_args();data=Path(args.data) + r=torch.load(args.checkpoint,map_location='cpu',weights_only=False) + dest=Path(args.out)/('step-'+str(r['step']));dest.mkdir(parents=True,exist_ok=True) + if (dest/'selection.json').exists():print((dest/'selection.json').read_text());return + save_file({k:v.to(torch.bfloat16).contiguous() for k,v in r['model'].items()},str(dest/'model.safetensors'), + metadata={'step':str(r['step']),'random_initialization':'true'}) + (dest/'config.json').write_text(json.dumps(r['config'],indent=2)) + info={k:r[k] for k in ['step','processed_tokens','response_tokens','training_seconds','random_initialization']} + (dest/'checkpoint-info.json').write_text(json.dumps(info,indent=2));del r + print(json.dumps({'frozen':str(dest),'step':info['step']}),flush=True);scores={};digest=None + for split in ['validation','development']: + if not (data/(split+'.jsonl')).exists():continue + with (dest/(split+'.log')).open('w') as log: + subprocess.run([sys.executable,'-u','-m','tinyquery.evaluate','--checkpoint',str(dest/'model.safetensors'), + '--tokenizer',str(data/'tokenizer.json'),'--data',str(data/(split+'.jsonl')), + '--out',str(dest/(split+'.jsonl'))],stdout=log,stderr=subprocess.STDOUT,check=True) + summary=json.loads((dest/(split+'.summary.json')).read_text());scores[split]=summary['metrics']['success'] + digest=summary['checkpoint_sha256'];print(json.dumps({'split':split,'metrics':summary['metrics']}),flush=True) + selection={'checkpoint':str(dest/'model.safetensors'),'step':info['step'],'checkpoint_sha256':digest, + 'development_scores':scores,'selection_score':sum(v['rate'] for v in scores.values())/len(scores), + 'rule':'Equal mean of full validation and development task success. Test/manual excluded.'} + (dest/'selection.json').write_text(json.dumps(selection,indent=2));print(json.dumps(selection),flush=True) + + +if __name__=='__main__':main() diff --git a/tinyquery/concrete.py b/tinyquery/concrete.py new file mode 100644 index 0000000000000000000000000000000000000000..2615d796d96933de7c2882553993cbbec8457066 --- /dev/null +++ b/tinyquery/concrete.py @@ -0,0 +1,78 @@ +"""Directly paraphrase concrete training scenarios, keeping the original reference action.""" +import argparse +import concurrent.futures +import json +import random +import re +import time +import urllib.request +from pathlib import Path +from tinyquery.data import compact,serialize +from tinyquery.recipes import LANGUAGES + + +def paraphrase(row,base,seed): + task={'schema':row['context']['schema'],'backend':row['backend'],'reference_request':row['question'], + 'reference_action':row['target']} + prompt=('Write one natural user request in each of four styles that has exactly the reference meaning. ' + 'Use conversational phrasing, not instructions about writing SQL. Keep every numeric bound, ' + 'literal value, selected field, sorting and limit unchanged. Do not add a new condition. ' + 'You may use ordinary English/Hindi words for obvious schema fields, but preserve database ' + 'literal values exactly. Output JSON keys en, noisy_en, hi, hinglish with string values. ' + 'hi must be Devanagari Hindi, hinglish Romanized Hindi, noisy_en imperfect English. Task: '+compact(task)) + payload={'model':'Qwen/Qwen3.8-27B-FP8','messages':[{'role':'user','content':prompt}], + 'max_tokens':500,'temperature':.7,'seed':seed,'response_format':{'type':'json_object'}, + 'chat_template_kwargs':{'enable_thinking':False}} + request=urllib.request.Request(base+'/chat/completions',data=compact(payload).encode(),headers={'Content-Type':'application/json'}) + with urllib.request.urlopen(request,timeout=180) as response: result=json.load(response) + parsed=json.loads(result['choices'][0]['message']['content']) + accepted=[] + for lang in LANGUAGES: + question=parsed.get(lang) + if not isinstance(question,str) or not 5'+compact(context)+'\n<|user|>'+question+'\n<|assistant|>' + + +def obj_schema(properties,required=None): + return {'type':'object','properties':properties,'required':list(properties) if required is None else required, + 'additionalProperties':False} + + +def tool(name,description,properties,required=None): + return {'name':name,'description':description,'inputSchema':obj_schema(properties,required)} + + +def make_tools(backend,rng,split,project_id): + prefix=rng.choice(['','db.','database.']) if split=='train' else ('warehouse.' if split=='test' else 'store.') + string={'type':'string'} + query_key='query' if backend=='supabase' else rng.choice(['sql','query']) + query_name=prefix+('execute_sql' if backend=='supabase' else rng.choice(['query','run_query','execute_sql'])) + scoped=backend!='supabase' or rng.random()<0.5 + project={} if scoped else {'project_id':string} + tools=[tool(query_name,'Execute a read-only '+('PostgreSQL' if backend=='supabase' else 'MySQL')+' SELECT query.', + {**project,query_key:string}), + tool(prefix+'list_tables','List database tables and their columns.', + {**project,'schemas':{'type':'array','items':string}},list(project))] + if backend=='mysql': + tools.append(tool(prefix+'describe_table','Inspect columns and types for one table.', + {**project,'table':string})) + return tools,query_key,project,scoped + + +def schema_for(domain,index): + h=int(hashlib.sha256(domain.encode()).hexdigest()[:8],16) + return {'table':domain,'column':TEXT_COLS[(h+index)%len(TEXT_COLS)], + 'numeric':NUM_COLS[(h+index)%len(NUM_COLS)],'category':CAT_COLS[(h+index)%len(CAT_COLS)], + 'date_column':DATE_COLS[(h+index)%len(DATE_COLS)],'parent':domain+'_groups', + 'foreign':'group_id','label':'label'} + + +def ddls(s): + return [f"CREATE TABLE {s['parent']} (id INTEGER PRIMARY KEY, {s['label']} TEXT);", + f"CREATE TABLE {s['table']} (id INTEGER PRIMARY KEY, {s['column']} TEXT, {s['numeric']} REAL, " + f"{s['category']} TEXT, {s['date_column']} DATE, {s['foreign']} INTEGER REFERENCES {s['parent']}(id));"] + + +def quote(value): return "'"+str(value).replace("'","''")+"'" + + +def gold_sql(op,s,backend): + t,c,n,g,d=s['table'],s['column'],s['numeric'],s['category'],s['date_column'] + v,o=quote(s['value']),quote(s['other']) + x,y,k=s['number'],s['upper'],s['limit'] + base=f'SELECT * FROM {t}' + queries={ + 'all':base,'project':f'SELECT {c} FROM {t}', + 'eq':f'{base} WHERE {g} = {v}','project_eq':f'SELECT {c} FROM {t} WHERE {g} = {v}', + 'gt':f'{base} WHERE {n} > {x}','lt':f'{base} WHERE {n} < {x}', + 'gte':f'{base} WHERE {n} >= {x}','lte':f'{base} WHERE {n} <= {x}', + 'between':f'{base} WHERE {n} BETWEEN {x} AND {y}', + 'and':f'{base} WHERE {g} = {v} AND {n} > {x}', + 'or':f'{base} WHERE {g} IN ({v}, {o})', + 'null':f'{base} WHERE {c} IS NULL','not_null':f'{base} WHERE {c} IS NOT NULL', + 'count':f'SELECT COUNT(*) FROM {t}','count_eq':f'SELECT COUNT(*) FROM {t} WHERE {g} = {v}', + 'sum':f'SELECT SUM({n}) FROM {t}','avg':f'SELECT AVG({n}) FROM {t}', + 'max':f'SELECT MAX({n}) FROM {t}','min':f'SELECT MIN({n}) FROM {t}', + 'distinct':f'SELECT DISTINCT {g} FROM {t}', + 'sort_desc':f'{base} ORDER BY {n} DESC','sort_asc':f'{base} ORDER BY {n} ASC', + 'top':f'{base} ORDER BY {n} DESC LIMIT {k}','bottom':f'{base} ORDER BY {n} ASC LIMIT {k}', + 'group_count':f'SELECT {g}, COUNT(*) FROM {t} GROUP BY {g}', + 'group_sum':f'SELECT {g}, SUM({n}) FROM {t} GROUP BY {g}', + 'having':f'SELECT {g}, COUNT(*) FROM {t} GROUP BY {g} HAVING COUNT(*) > {x}', + 'date_after':f'{base} WHERE {d} >= '+quote(s['date']), + 'year':f'{base} WHERE '+(f'YEAR({d})' if backend=='mysql' else f'EXTRACT(YEAR FROM {d})')+f" = {s['year']}", + 'month':f'{base} WHERE '+(f'MONTH({d})' if backend=='mysql' else f'EXTRACT(MONTH FROM {d})')+f" = {s['month']}", + 'contains':f'{base} WHERE LOWER({c}) LIKE LOWER('+quote('%'+s['value']+'%')+')', + 'join':f"SELECT {t}.{c}, {s['parent']}.{s['label']} FROM {t} JOIN {s['parent']} ON {t}.{s['foreign']} = {s['parent']}.id", + 'join_filter':f"SELECT {t}.{c} FROM {t} JOIN {s['parent']} ON {t}.{s['foreign']} = {s['parent']}.id WHERE {s['parent']}.{s['label']} = {v}", + } + return queries[op]+';' + + +def fixture(s,seed=0): + rng=random.Random(seed) + db=sqlite3.connect(':memory:') + db.create_function('YEAR',1,lambda d: None if d is None else int(str(d)[:4])) + db.create_function('MONTH',1,lambda d: None if d is None else int(str(d)[5:7])) + for ddl in ddls(s): db.execute(ddl) + db.executemany(f"INSERT INTO {s['parent']} VALUES (?,?)",[(1,s['value']),(2,s['other']),(3,'West')]) + vals=[s['number']-1,s['number'],s['number']+1,s['upper'],s['upper']+1] + rows=[] + for i in range(24): + rows.append((i+1, None if i%9==0 else rng.choice(['Asha','Ravi','item '+s['value'],'Other']), + (vals[i%len(vals)] if i<6 else rng.choice(vals)+i*0.001) if i%6 else None, + rng.choice([s['value'],s['other'],'Other',None]), + rng.choice([s['date'],f"{s['year']}-01-01",'2022-07-18',None]),rng.choice([1,2,3,None]))) + db.executemany(f"INSERT INTO {s['table']} VALUES (?,?,?,?,?,?)",rows) + db.execute('PRAGMA query_only=ON') + return db + + +def validate_sql(sql,backend,s): + import sqlglot + from sqlglot import exp + parsed=sqlglot.parse(sql,read='postgres' if backend=='supabase' else 'mysql') + if len(parsed)!=1 or not isinstance(parsed[0],exp.Select): raise ValueError('Not one SELECT') + result=[] + for seed in (1,7): + db=fixture(s,seed) + translated=sqlite_sql(parsed[0]) + try: result.append(db.execute(translated).fetchall()) + except Exception as exc: raise ValueError(f'{sql} -> {translated}: {exc}') from exc + finally: db.close() + return result + + +def validate_context_sql(sql,backend,schema): + """Compile against the supplied fictitious schema during offline evaluation/auditing.""" + import sqlglot + db=sqlite3.connect(':memory:');db.create_function('YEAR',1,lambda x:0);db.create_function('MONTH',1,lambda x:0) + try: + for ddl in schema:db.execute(ddl) + db.execute('PRAGMA query_only=ON') + db.execute(sqlite_sql(sqlglot.parse_one(sql,read='postgres' if backend=='supabase' else 'mysql'))) + finally:db.close() + + +def sqlite_sql(tree): + from sqlglot import exp + def rewrite(node): + if isinstance(node,exp.Extract): + unit=str(node.this).upper() + if unit in ('YEAR','MONTH'): + return exp.Cast(this=exp.Anonymous(this='STRFTIME',expressions=[ + exp.Literal.string('%Y' if unit=='YEAR' else '%m'),node.expression.copy()]), + to=exp.DataType.build('INTEGER')) + return node + return tree.copy().transform(rewrite).sql(dialect='sqlite') + + +def load_templates(path): + templates={} + for line in Path(path).read_text().splitlines(): + row=json.loads(line); templates[row['op']]=row['templates'] + missing=set(RECIPES)-set(templates) + if missing: raise ValueError(f'Missing teacher recipes: {sorted(missing)}') + return templates + + +def choose_template(templates,op,lang,split,rng): + pool=templates[op][lang] + if len(pool)<4: raise ValueError(f'Need four templates for {op}/{lang}, have {len(pool)}') + indices=list(range(len(pool)-2)) if split=='train' else [len(pool)-(2 if split=='validation' else 1)] + idx=rng.choice(indices) + return pool[idx],idx + + +def build_record(op,domain,index,backend,split,templates,rng): + s=schema_for(domain,index) + number=rng.choice([0,1,2,3,5,10,20,50,100,200,500,1000,1500,5000,10000]) + s.update(number=number,upper=number+rng.choice([10,50,100,500]),limit=rng.choice([1,3,5,10,20]), + value=rng.choice(VALUES),other=rng.choice(VALUES),date=f'202{rng.randrange(2,7)}-{rng.randrange(1,13):02d}-15', + year=rng.randrange(2021,2027),month=rng.randrange(1,13),city=rng.choice(['Delhi','Mumbai','Lucknow','Jaipur']), + unit=rng.choice(['celsius','fahrenheit']),query=rng.choice(['row level security','SQL joins','API authentication','database backups','indexes']), + path=rng.choice(['/docs/README.md','/reports/sales.csv','/notes/setup.txt','/config/app.json']), + ticket_id='TKT-'+str(rng.randrange(100,10000)),count=rng.randrange(0,150)) + project_id='demo_'+domain + tools,key,project,scoped=make_tools(backend,rng,split,project_id) + project_args={k:project_id for k in project} + context={'backend':backend,'project_id':project_id,'schema':ddls(s), 'tools':tools, + 'policy':'Read-only database access. Use provided tools. Ask when required information is missing.'} + if op in SQL_OPS: + sql=gold_sql(op,s,backend) + validate_sql(sql,backend,s) + target={'action':'call','name':tools[0]['name'],'arguments':{**project_args,key:sql}} + elif op in ('list_tables','missing_schema'): + if op=='missing_schema': context['schema']=[] + target={'action':'call','name':tools[1]['name'],'arguments':{**project_args,'schemas':['public']}} + elif op in ('describe','sql_error'): + if op=='sql_error': + context['history']=[{'role':'tool','isError':True,'content':'Unknown column in previous query. Inspect the schema.'}] + context['schema']=[] + target=({'action':'call','name':tools[2]['name'],'arguments':{**project_args,'table':domain}} + if backend=='mysql' else + {'action':'call','name':tools[1]['name'],'arguments':{**project_args,'schemas':['public']}}) + elif op in ('ambiguous','missing_value','write'): + target={'action':'clarify','question':{'ambiguous':'What does best mean: which column and order?', + 'missing_value':f"Which value should {s['category']} equal?",'write':'This connection is read-only. Would you like a SELECT query instead?'}[op]} + elif op=='final': + context['history']=[{'role':'tool','content':{'row_count':s['count']}}] + target={'action':'answer','text':f"The tool returned {s['count']} rows."} + else: + definitions={ + 'weather':('get_weather','Get current weather for a city.',{'city':{'type':'string'},'unit':{'type':'string','enum':['celsius','fahrenheit']}}, {'city':s['city'],'unit':s['unit']}), + 'search':('search_docs','Search documentation for a query.',{'query':{'type':'string'}},{'query':s['query']}), + 'read_file':('read_file','Read text from a file path.',{'path':{'type':'string'}},{'path':s['path']}), + 'ticket':('get_ticket','Retrieve a support ticket by its ID.',{'ticket_id':{'type':'string'}},{'ticket_id':s['ticket_id']}), + } + generic=[] + for kind,(name,desc,properties,args) in definitions.items(): + prefix=rng.choice(['','tools.','app.']) if split=='train' else ('external.' if split=='test' else 'service.') + generic.append(tool(prefix+name,desc,properties)) + if kind==op: target={'action':'call','name':prefix+name,'arguments':args} + tools.extend(generic) + context['schema']=[] + rng.shuffle(tools) + semantic_id=hashlib.sha256(compact([split,backend,s,op,target]).encode()).hexdigest()[:20] + records=[] + for lang in LANGUAGES: + template,template_index=choose_template(templates,op,lang,split,rng) + question=template.format(**s) + answer=dict(target) + if lang=='hi' and answer['action']=='clarify': + answer['question']={'ambiguous':'सबसे अच्छा से आपका क्या मतलब है? कौन सा कॉलम और क्रम?', + 'missing_value':f"{s['category']} का कौन सा मान चाहिए?",'write':'यह कनेक्शन केवल पढ़ने के लिए है। क्या आपको SELECT क्वेरी चाहिए?'}[op] + elif lang=='hinglish' and answer['action']=='clarify': + answer['question']={'ambiguous':'Best ka matlab kya hai? Kaunsa column aur order?', + 'missing_value':f"{s['category']} ki kaunsi value chahiye?",'write':'Yeh connection read-only hai. Kya SELECT query chahiye?'}[op] + elif answer['action']=='answer': + if lang=='hi': answer['text']=f"टूल ने {s['count']} पंक्तियाँ लौटाईं।" + elif lang=='hinglish': answer['text']=f"Tool ne {s['count']} rows return ki." + records.append({'id':semantic_id+'_'+lang,'scenario_id':semantic_id,'split':split,'domain':domain, + 'language':lang,'backend':backend,'operation':op,'slots':s,'context':context, + 'question':question,'target':answer,'prompt':serialize(context,question), + 'response':compact(answer),'template_index':template_index, + 'provenance':'programmatic semantics + Qwen3.8-27B-FP8 language template'}) + return records + + +def main(): + p=argparse.ArgumentParser(); p.add_argument('--templates',required=True) + p.add_argument('--out',default='data/tinyquery'); p.add_argument('--scenarios',type=int,default=18000) + args=p.parse_args(); out=Path(args.out); out.mkdir(parents=True,exist_ok=True) + templates=load_templates(args.templates); stats={}; global_prompts={} + for split,count in [('train',args.scenarios),('validation',300),('test',300)]: + rng=random.Random({'train':42,'validation':43,'test':44}[split]); seen=set(); counts={} + with (out/(split+'.jsonl')).open('w') as stream: + for i in range(count): + op=list(RECIPES)[i%len(RECIPES)]; domain=rng.choice(DOMAINS[split]) + for record in build_record(op,domain,i,rng.choice(['mysql','supabase']),split,templates,rng): + key=hashlib.sha256(record['prompt'].encode()).hexdigest() + if key in seen: continue + if key in global_prompts and global_prompts[key]!=split: raise ValueError('Cross-split prompt leakage') + seen.add(key); global_prompts[key]=split + stream.write(json.dumps(record,ensure_ascii=False)+'\n') + counts[op]=counts.get(op,0)+1 + if i%1000==0: print(split,i,flush=True) + stats[split]={'examples':len(seen),'operations':counts,'domains':DOMAINS[split]} + stats['sql_validation_method']='Dialect parsing plus transpiled SQLite execution on two fixtures; not native MySQL/Postgres execution.' + (out/'dataset-stats.json').write_text(json.dumps(stats,indent=2)) + print(json.dumps(stats),flush=True) + + +if __name__=='__main__': main() diff --git a/tinyquery/evaluate.py b/tinyquery/evaluate.py new file mode 100644 index 0000000000000000000000000000000000000000..64cdce51e55a3cbe317b0f2c6ca28d6ff4b47b34 --- /dev/null +++ b/tinyquery/evaluate.py @@ -0,0 +1,162 @@ +"""Raw greedy tool-calling evaluation; SQL equivalence is explicitly SQLite-based.""" +import argparse +from collections import Counter,defaultdict +import contextlib +import hashlib +import json +import time +from pathlib import Path +import torch +import jsonschema +from tokenizers import Tokenizer +from tinyquery.model import TinyQuery,Config +from tinyquery.data import validate_sql,validate_context_sql,SQL_OPS + + +def parse_action(text): + def unique(pairs): + result={} + for key,value in pairs: + if key in result:raise ValueError('Duplicate JSON key: '+key) + result[key]=value + return result + return json.loads(text,object_pairs_hook=unique) + + +def check_action(action,context): + if not isinstance(action,dict): raise ValueError('Action must be an object') + kind=action.get('action') + if kind=='call': + if set(action)!={'action','name','arguments'}: raise ValueError('Unexpected action keys') + tool=next((t for t in context['tools'] if t['name']==action.get('name')),None) + if tool is None: raise ValueError('Unknown tool') + jsonschema.validate(action.get('arguments'),tool['inputSchema']) + elif kind in ('clarify','answer'): + key='question' if kind=='clarify' else 'text' + if set(action)!={'action',key} or not isinstance(action[key],str) or not action[key].strip(): + raise ValueError('Invalid textual action') + else: raise ValueError('Unknown action') + return action + + +def mcp_request(action,context,request_id=1): + check_action(action,context) + if action['action']!='call': raise ValueError('Only tool actions map to tools/call') + # Scope is an inference input, not a string the model may freely substitute. + if 'project_id' in action['arguments'] and action['arguments']['project_id']!=context.get('project_id'): + raise ValueError('Predicted project_id differs from the supplied project scope') + tool=next(t for t in context['tools'] if t['name']==action['name']) + description=tool.get('description','') + for key,value in action['arguments'].items(): + if key=='sql' or (key=='query' and 'SELECT query' in description): + import sqlglot + from sqlglot import exp + trees=sqlglot.parse(value,read='mysql' if context['backend']=='mysql' else 'postgres') + if len(trees)!=1 or not isinstance(trees[0],exp.Select):raise ValueError('Expected one read-only SELECT') + tree=trees[0] + if tree.find(exp.Into) or any(isinstance(n,(exp.DML,exp.DDL)) for n in tree.walk()): + raise ValueError('Data-changing SQL is not supported') + known={sqlglot.parse_one(ddl).this.this.name for ddl in context.get('schema',[])} + if any(table.name not in known for table in tree.find_all(exp.Table)): + raise ValueError('SQL references a table absent from the supplied schema') + return {'jsonrpc':'2.0','id':request_id,'method':'tools/call', + 'params':{'name':action['name'],'arguments':action['arguments']}} + + +def score(row,text): + result={'json_valid':False,'schema_valid':False,'action_correct':False,'tool_correct':False, + 'arguments_exact':False,'sql_equivalent':None,'success':False} + if row['target']['action']!='call': + result['tool_correct']=None; result['arguments_exact']=None + if row['operation'] in SQL_OPS: result['sql_equivalent']=False + try: + parsed=parse_action(text); result['json_valid']=True + if isinstance(parsed,dict): result['action_correct']=parsed.get('action')==row['target']['action'] + check_action(parsed,row['context']); result['schema_valid']=True + gold=row['target']; result['action_correct']=parsed['action']==gold['action'] + if not result['action_correct']: return result + if gold['action']!='call': + key='question' if gold['action']=='clarify' else 'text' + # Action accuracy and wording exact-match are separate; generic clarification is not semantic proof. + result['text_exact']=parsed[key]==gold[key] + result['success']=result['text_exact'] + return result + result['tool_correct']=parsed['name']==gold['name'] + result['arguments_exact']=parsed['arguments']==gold['arguments'] + sqlkey=next((k for k in ('sql','query') if k in gold['arguments'] and str(gold['arguments'][k]).startswith('SELECT ')),None) + if sqlkey: + # Backend/project arguments must also match; equal SQL against a wrong project is not a pass. + other_correct={k:v for k,v in parsed['arguments'].items() if k!=sqlkey}=={k:v for k,v in gold['arguments'].items() if k!=sqlkey} + validate_context_sql(parsed['arguments'][sqlkey],row['backend'],row['context']['schema']) + actual=validate_sql(parsed['arguments'][sqlkey],row['backend'],row['slots']) + expected=validate_sql(gold['arguments'][sqlkey],row['backend'],row['slots']) + ordered=row['operation'] in ['sort_asc','sort_desc','top','bottom'] + normalize=lambda a:a if ordered else sorted(a,key=repr) + equivalent=all(normalize(a)==normalize(b) for a,b in zip(actual,expected)) + result['sql_equivalent']=equivalent + result['success']=result['tool_correct'] and other_correct and equivalent + else: result['success']=result['tool_correct'] and result['arguments_exact'] + except Exception as exc: result['error']=str(exc)[:350] + return result + + +def load_model(checkpoint,device): + path=Path(checkpoint) + if path.suffix=='.safetensors': + from safetensors.torch import load + c=Config(**json.loads((path.parent/'config.json').read_text())) + raw=path.read_bytes(); state=load(raw) + model=TinyQuery(c); model.load_state_dict(state) + model.checkpoint_sha256=hashlib.sha256(raw).hexdigest();del raw,state + else: + state=torch.load(path,map_location='cpu',weights_only=False) + model=TinyQuery(Config(**state['config'])); model.load_state_dict(state['model']); del state + model=model.to(device) + if device=='cuda': model=model.to(torch.bfloat16) + return model.eval() + + +def aggregate(results): + metrics={} + for key in ['json_valid','schema_valid','action_correct','tool_correct','arguments_exact','sql_equivalent','success']: + entries=[r[key] for r in results if r.get(key) is not None] + metrics[key]={'correct':sum(entries),'total':len(entries),'rate':sum(entries)/max(1,len(entries))} + return metrics + + +def main(): + p=argparse.ArgumentParser(); p.add_argument('--checkpoint',required=True); p.add_argument('--tokenizer',required=True) + p.add_argument('--data',required=True); p.add_argument('--out',required=True); p.add_argument('--limit',type=int,default=0) + p.add_argument('--batch',type=int,default=16); p.add_argument('--tokens',type=int,default=160) + args=p.parse_args(); device='cuda' if torch.cuda.is_available() else ('mps' if torch.backends.mps.is_available() else 'cpu') + torch.set_num_threads(8); model=load_model(args.checkpoint,device); tokenizer=Tokenizer.from_file(args.tokenizer) + rows=[json.loads(l) for l in Path(args.data).read_text().splitlines()] + if args.limit: + import random + random.Random(777).shuffle(rows); rows=rows[:args.limit] + # Sort only for efficient padding, never alter which examples are evaluated. + encoded=[(r,tokenizer.encode(r['prompt']).ids) for r in rows]; encoded.sort(key=lambda item:len(item[1])) + out=Path(args.out); out.parent.mkdir(parents=True,exist_ok=True); results=[]; start=time.time() + groups=defaultdict(list) + with out.open('w') as stream: + for i in range(0,len(encoded),args.batch): + batch=encoded[i:i+args.batch] + with torch.inference_mode(): + outputs=model.generate_batch([ids for _,ids in batch],tokenizer.token_to_id('<|end|>'),max_new_tokens=args.tokens) + for (row,_),ids in zip(batch,outputs): + text=tokenizer.decode(ids,skip_special_tokens=True) + metrics=score(row,text); results.append(metrics) + for field in ['language','backend','operation']: groups[field+':'+row[field]].append(metrics) + entry={'id':row['id'],'language':row['language'],'backend':row['backend'],'operation':row['operation'], + 'question':row['question'],'expected':row['target'],'output':text,'metrics':metrics} + stream.write(json.dumps(entry,ensure_ascii=False)+'\n') + stream.flush() + print(json.dumps({'evaluated':len(results),'seconds':time.time()-start,'success':sum(r['success'] for r in results)/len(results)}),flush=True) + summary={'examples':len(results),'seconds':time.time()-start,'decoding':'raw greedy, no repair, no teacher fallback', + 'checkpoint_sha256':getattr(model,'checkpoint_sha256',None), + 'sql_metric':'Compile against the supplied schema, then compare results on two generated SQLite fixtures after dialect adaptation; not native MySQL/PostgreSQL execution.', + 'metrics':aggregate(results),'groups':{k:aggregate(v) for k,v in groups.items()}} + out.with_suffix('.summary.json').write_text(json.dumps(summary,indent=2)); print(json.dumps(summary),flush=True) + + +if __name__=='__main__': main() diff --git a/tinyquery/export.py b/tinyquery/export.py new file mode 100644 index 0000000000000000000000000000000000000000..6633505f0b6c84015aa5491cf54549fa8c3a28d0 --- /dev/null +++ b/tinyquery/export.py @@ -0,0 +1,50 @@ +"""Export only inference weights and public project code; never upload credentials or remote config.""" +import argparse +import hashlib +import json +from pathlib import Path +import shutil +import torch +from safetensors.torch import save_file +from tinyquery.model import Config,TinyQuery + + +def main(): + p=argparse.ArgumentParser(); p.add_argument('--checkpoint',required=True); p.add_argument('--data',required=True) + p.add_argument('--out',required=True); args=p.parse_args() + source=Path(args.checkpoint); data=Path(args.data); out=Path(args.out); out.mkdir(parents=True,exist_ok=True) + if source.suffix=='.safetensors': + config=json.loads((source.parent/'config.json').read_text()) + shutil.copy2(source,out/'model.safetensors') + info=source.parent/'checkpoint-info.json' + if not info.exists():info=source.parent/'best-info.json' + if info.exists(): shutil.copy2(info,out/'checkpoint-info.json') + else: + checkpoint=torch.load(source,map_location='cpu',weights_only=False) + config=checkpoint['config'] + state={k:v.to(torch.bfloat16).contiguous() for k,v in checkpoint['model'].items()} + save_file(state,str(out/'model.safetensors')) + info={k:checkpoint[k] for k in ['step','processed_tokens','response_tokens','training_seconds','random_initialization']} + (out/'checkpoint-info.json').write_text(json.dumps(info,indent=2)) + (out/'config.json').write_text(json.dumps(config,indent=2)) + shutil.copy2(data/'tokenizer.json',out/'tokenizer.json') + package=Path(__file__).parent + shutil.copytree(package,out/'tinyquery',dirs_exist_ok=True,ignore=shutil.ignore_patterns('__pycache__','*.pyc')) + shutil.copy2(package/'requirements-inference.txt',out/'requirements.txt') + for name in ['tokenization.json','grounding-stats.json','split-audit.json']: + if (data/name).exists(): shutil.copy2(data/name,out/name) + model=TinyQuery(Config(**config)); count=sum(p.numel() for p in model.parameters()) + from safetensors.torch import load_file + model.load_state_dict(load_file(str(out/'model.safetensors')),strict=True) + manifest={'parameters':count,'format':'Custom native PyTorch; see tinyquery/model.py, not a Transformers AutoModel checkpoint', + 'random_initialization':True,'files':{}} + for file in sorted(out.rglob('*')): + if file.is_file() and file.name!='manifest.json': + digest=hashlib.sha256() + with file.open('rb') as stream: + for block in iter(lambda:stream.read(4*1024*1024),b''): digest.update(block) + manifest['files'][str(file.relative_to(out))]={'bytes':file.stat().st_size,'sha256':digest.hexdigest()} + (out/'manifest.json').write_text(json.dumps(manifest,indent=2)); print(json.dumps({'parameters':count,'files':len(manifest['files'])})) + + +if __name__=='__main__': main() diff --git a/tinyquery/filter_templates.py b/tinyquery/filter_templates.py new file mode 100644 index 0000000000000000000000000000000000000000..29d8f2d888daf2bdc6d3d39a8a2dccb3052d9f6f --- /dev/null +++ b/tinyquery/filter_templates.py @@ -0,0 +1,35 @@ +"""Conservatively exclude training template variants flagged by the teacher audit.""" +import argparse +from collections import Counter +import json +from pathlib import Path + + +def main(): + p=argparse.ArgumentParser();p.add_argument('--data',required=True);p.add_argument('--audit',required=True) + args=p.parse_args();base=Path(args.data);bad=set();accepted=0;jobs=0 + for line in Path(args.audit).read_text().splitlines(): + row=json.loads(line);jobs+=1;accepted+=len(row['accepted']) + for record in row['rejected']: + # IDs are audit___; both names may contain underscores. + tail=record['id'][6:] + import re + match=re.fullmatch(r'(.+)_(\d+)_(en|noisy_en|hi|hinglish)',tail) + assert match,record['id'] + op,index,language=match.groups();bad.add((op,language,int(index))) + removed=Counter();kept=0;source=base/'train.jsonl';temp=base/'train.filtered.jsonl' + with source.open() as inp,temp.open('w') as out: + for line in inp: + row=json.loads(line);key=(row['operation'],row['language'],row.get('template_index')) + if key in bad and row['provenance'].startswith('programmatic semantics + Qwen3.8-27B-FP8 language template'): + removed['/'.join(map(str,key))]+=1 + else:out.write(line);kept+=1 + temp.replace(source) + report={'audit_jobs':jobs,'accepted_template_instances':accepted,'flagged_template_variants':len(bad), + 'excluded_training_rows':sum(removed.values()),'remaining_training_rows':kept,'exclusions':dict(removed), + 'policy':'Conservative exclusion after SQL round-trip disagreement; disagreement alone does not prove the teacher was correct. Validation, test, manual, direct verified paraphrases and separately authored contrasts remain unchanged.', + 'limitation':'Earlier curriculum stages trained on these rows before the audit; this filtering does not undo earlier exposure.'} + (base/'template-filter.json').write_text(json.dumps(report,indent=2));print(json.dumps(report)) + + +if __name__=='__main__':main() diff --git a/tinyquery/grounding_data.py b/tinyquery/grounding_data.py new file mode 100644 index 0000000000000000000000000000000000000000..55806de3687b63a9caa55830650009668e0ca29e --- /dev/null +++ b/tinyquery/grounding_data.py @@ -0,0 +1,94 @@ +"""Counter identifier memorization with randomized, semantically equivalent training schemas.""" +import argparse +import copy +import hashlib +import json +import random +import re +import shutil +import string +from pathlib import Path +from tinyquery.data import SQL_OPS,ddls,gold_sql,serialize,compact + +CHUNKS=[] +PLAIN=False + +def random_id(rng,prefix): + if PLAIN and prefix in ['t','g','c'] and rng.random()<.85: + from sqlglot import Tokenizer as SQLTokenizer + while True: + value=''.join(rng.choices(CHUNKS,k=rng.choice([3,4,5]))) if CHUNKS and rng.random()<.8 else ''.join(rng.choices(string.ascii_lowercase,k=rng.choice([6,7,8]))) + if len(value)>=6 and value.upper() not in SQLTokenizer.KEYWORDS: return value + if CHUNKS and rng.random()<.8: + return prefix+'_'+''.join(rng.choices(CHUNKS,k=rng.choice([2,3,4]))) + return prefix+'_'+''.join(rng.choices(string.ascii_lowercase,k=rng.choice([4,5,6,7]))) + + +def main(): + global PLAIN + p=argparse.ArgumentParser(); p.add_argument('--source',default='data/tinyquery');p.add_argument('--out',default='data/tinyquery-v2') + p.add_argument('--token-chunks',action='store_true'); p.add_argument('--seed',type=int,default=99032) + p.add_argument('--extra',help='Replay an earlier augmentation corpus, deduplicated by prompt') + p.add_argument('--plain-identifiers',action='store_true') + args=p.parse_args(); source=Path(args.source); out=Path(args.out); out.mkdir(parents=True,exist_ok=True) + PLAIN=args.plain_identifiers + for name in ['validation.jsonl','test.jsonl','manual.jsonl','tokenizer.json']: + shutil.copy2(source/name,out/name) + if args.token_chunks: + from tokenizers import Tokenizer + tokenizer=Tokenizer.from_file(str(source/'tokenizer.json')) + CHUNKS.extend(sorted({tokenizer.decode([i]) for i in range(tokenizer.get_vocab_size()) + if re.fullmatch('[a-z]{1,8}',tokenizer.decode([i]))})) + rng=random.Random(args.seed); counts={'original':0,'grounding':0,'discovery':0,'replay':0}; seen=set() + with (out/'train.jsonl').open('w') as stream: + def emit(r,kind): + key=hashlib.sha256(r['prompt'].encode()).hexdigest() + if key in seen: return + seen.add(key); counts[kind]+=1; stream.write(json.dumps(r,ensure_ascii=False)+'\n') + with (source/'train.jsonl').open() as original: + for line in original: emit(json.loads(line),'original') + with (source/'train-base.jsonl').open() as original: + for index,line in enumerate(original): + r=json.loads(line) + if re.search('[\u0600-\u06ff]',r['question']): continue + s=r['slots']; mapping={} + if r['operation'] in SQL_OPS and rng.random()<.85: + for field in ['table','parent']+(['column','numeric','category','date_column','foreign','label'] if rng.random()<.6 else []): + old=s[field]; s[field]=random_id(rng,{'table':'t','parent':'g'}.get(field,'c')) + mapping[old]=s[field] + if rng.random()<.5: + for field in ['value','other']: + old=str(s[field]); s[field]=mapping.get(old) or ''.join(rng.choices(string.ascii_letters,k=7)) + mapping[old]=s[field] + # All source templates contain the explicit identifier/value slots being renamed. + pattern=re.compile(r'(?1: + mask=torch.arange(k.shape[2],device=x.device)[None,:]<=torch.arange(offset,offset+t,device=x.device)[:,None] + y=F.scaled_dot_product_attention(q,k,v,attn_mask=mask,is_causal=causal,enable_gqa=True) + y=y.transpose(1,2).contiguous().view(b,t,-1) + return self.out(y),(k,v) if use_cache else None + + +class Block(nn.Module): + def __init__(self,c): + super().__init__(); self.norm1=RMSNorm(c.width); self.attn=Attention(c); self.norm2=RMSNorm(c.width) + self.gate_up=nn.Linear(c.width,2*c.hidden,bias=False); self.down=nn.Linear(c.hidden,c.width,bias=False) + def forward(self,x,cos,sin,past=None,pad_mask=None,use_cache=False): + a,cache=self.attn(self.norm1(x),cos,sin,past,pad_mask,use_cache) + x=x+a; gate,up=self.gate_up(self.norm2(x)).chunk(2,dim=-1) + return x+self.down(F.silu(gate)*up),cache + + +class TinyQuery(nn.Module): + def __init__(self,c): + super().__init__(); self.config=c + assert c.width%c.heads==0 and c.heads%c.kv_heads==0 and (c.width//c.heads)%2==0 + self.tokens=nn.Embedding(c.vocab_size,c.width) + self.blocks=nn.ModuleList([Block(c) for _ in range(c.layers)]) + self.norm=RMSNorm(c.width); self.action_head=nn.Linear(c.width,3,bias=False) + if c.copy_dim: + self.copy_query=nn.Linear(c.width,c.copy_dim,bias=False) + self.copy_key=nn.Linear(c.width,c.copy_dim,bias=False) + self.copy_gate=nn.Linear(c.width,1) + dim=c.width//c.heads + inv=1/(c.rope_theta**(torch.arange(0,dim,2,dtype=torch.float32)/dim)) + angles=torch.outer(torch.arange(c.context,dtype=torch.float32),inv) + self.register_buffer('rope_cos',angles.cos()[None,None,:,:],persistent=False) + self.register_buffer('rope_sin',angles.sin()[None,None,:,:],persistent=False) + self.apply(self._init) + if c.copy_dim: + nn.init.zeros_(self.copy_gate.weight); nn.init.constant_(self.copy_gate.bias,2.0) + for block in self.blocks: + nn.init.normal_(block.attn.out.weight,std=0.02/math.sqrt(2*c.layers)) + nn.init.normal_(block.down.weight,std=0.02/math.sqrt(2*c.layers)) + @staticmethod + def _init(m): + if isinstance(m,(nn.Linear,nn.Embedding)): nn.init.normal_(m.weight,std=0.02) + def forward(self,ids,targets=None,weights=None,boundaries=None,actions=None, + past=None,pad_mask=None,use_cache=False,last_only=False,prompt_weight=0.15): + length=ids.shape[1]; offset=0 if past is None else past[0][0].shape[2] + if offset+length>self.config.context: raise ValueError('Context limit exceeded') + x=self.tokens(ids) + cos=self.rope_cos[:,:,offset:offset+length,:].to(x.dtype) + sin=self.rope_sin[:,:,offset:offset+length,:].to(x.dtype) + caches=[] + for i,block in enumerate(self.blocks): + x,cache=block(x,cos,sin,None if past is None else past[i],pad_mask,use_cache) + if use_cache: caches.append(cache) + x=self.norm(x) + action_logits=None + if boundaries is not None: + selected=x[torch.arange(x.shape[0],device=x.device),boundaries] + action_logits=self.action_head(selected) + output=x[:,-1:,:] if last_only else x + logits=F.linear(output,self.tokens.weight) + copy_attention=None + if self.config.copy_dim: + keys=self.copy_key(x); source_ids=ids + if past is not None: + keys=torch.cat((past[-1][0],keys),dim=1) + source_ids=torch.cat((past[-1][1],ids),dim=1) + query=self.copy_query(output) + scores=(query@keys.transpose(-1,-2)).float()/math.sqrt(self.config.copy_dim) + query_positions=torch.arange(offset+length-output.shape[1],offset+length,device=ids.device) + allowed=(torch.arange(keys.shape[1],device=ids.device)[None,:]<=query_positions[:,None])[None,:,:] + allowed=allowed & (source_ids[:,None,:]!=0) + # Copy the supplied context/question, never recycle generated response text. + allowed=allowed & ((source_ids==3).cumsum(-1)==0)[:,None,:] + if pad_mask is not None: allowed=allowed & pad_mask[:,None,:] + copy_attention=scores.masked_fill(~allowed,-1e9).softmax(-1)*allowed + gate=self.copy_gate(output).float().sigmoid() + if use_cache: caches.append((keys,source_ids)) + if targets is None: + if copy_attention is not None: + probabilities=logits.float().softmax(-1)*gate + indices=source_ids[:,None,:].expand(-1,output.shape[1],-1) + probabilities=probabilities.scatter_add(-1,indices,copy_attention*(1-gate)) + logits=probabilities.clamp_min(1e-30).log() + return logits,caches,action_logits + if copy_attention is None: + losses=F.cross_entropy(logits.reshape(-1,logits.shape[-1]).float(),targets.reshape(-1), + ignore_index=-100,reduction='none').view_as(targets) + else: + safe_targets=targets.clamp_min(0) + generated=logits.float().log_softmax(-1).gather(-1,safe_targets[:,:,None]).squeeze(-1).exp() + copied=(copy_attention*(source_ids[:,None,:]==safe_targets[:,:,None])).sum(-1) + losses=-(gate.squeeze(-1)*generated+(1-gate.squeeze(-1))*copied).clamp_min(1e-30).log() + valid=targets!=-100 + response=(weights>0)&valid + token_weights=torch.where(response,1.0,prompt_weight)*valid + lm=(losses*token_weights).sum()/token_weights.sum().clamp_min(1) + auxiliary=F.cross_entropy(action_logits.float(),actions) if actions is not None else lm*0 + response_loss=(losses*response).sum()/response.sum().clamp_min(1) + return lm+0.05*auxiliary,torch.stack((lm.detach(),response_loss.detach(),auxiliary.detach())) + + @torch.no_grad() + def generate_batch(self,prompts,eos_id,pad_id=0,max_new_tokens=180): + self.eval(); device=next(self.parameters()).device + longest=max(map(len,prompts)) + if longest+max_new_tokens>self.config.context: + max_new_tokens=self.config.context-longest + if max_new_tokens<=0: raise ValueError('Prompt leaves no output space') + ids=torch.full((len(prompts),longest),pad_id,dtype=torch.long,device=device) + mask=torch.zeros_like(ids,dtype=torch.bool) + for i,p in enumerate(prompts): + ids[i,-len(p):]=torch.tensor(p,device=device); mask[i,-len(p):]=True + outputs=[[] for _ in prompts]; finished=torch.zeros(len(prompts),dtype=torch.bool,device=device) + past=None + for _ in range(max_new_tokens): + logits,past,_=self(ids,past=past,pad_mask=mask,use_cache=True,last_only=True) + next_ids=logits[:,-1].argmax(dim=-1) + done=finished.tolist() + for i,token in enumerate(next_ids.tolist()): + if not done[i]: outputs[i].append(token) + finished|=next_ids==eos_id + if finished.all(): break + ids=next_ids[:,None] + mask=torch.cat((mask,torch.ones((len(prompts),1),device=device,dtype=torch.bool)),dim=1) + return outputs diff --git a/tinyquery/native_sql.py b/tinyquery/native_sql.py new file mode 100644 index 0000000000000000000000000000000000000000..a27a9800896c0d8844ca5dbb62682b1c491db623 --- /dev/null +++ b/tinyquery/native_sql.py @@ -0,0 +1,108 @@ +"""Validate queries on disposable native MySQL 8/PostgreSQL 16 temporary tables.""" +import argparse +import json +import re +import time +from pathlib import Path +import sqlglot +from sqlglot import exp +from tinyquery.data import ddls,fixture,SQL_OPS + + +class NativeDB: + def __init__(self,backend): + self.backend=backend + if backend=='mysql': + import pymysql + self.conn=pymysql.connect(unix_socket='/run/mysqld/mysqld.sock',user='root',database='tinyquery',autocommit=True) + else: + import psycopg + self.conn=psycopg.connect(host='/var/run/postgresql',user='root',dbname='postgres',autocommit=True) + self.cursor=self.conn.cursor(); self.tables=[] + self.cursor.execute('SET SESSION max_execution_time=2000' if backend=='mysql' else "SET statement_timeout='2s'") + def setup(self,s,seed): + for table in reversed(self.tables): self.cursor.execute(f'DROP TABLE IF EXISTS {table}') + self.tables=[] + for ddl in ddls(s): + # Relationships are populated identically; MySQL temp tables cannot enforce foreign keys. + ddl=re.sub(r' REFERENCES [a-zA-Z_]+\(id\)','',ddl) + self.cursor.execute(ddl.replace('CREATE TABLE','CREATE TEMPORARY TABLE',1)) + self.tables=[s['parent'],s['table']] + source=fixture(s,seed) + for table in self.tables: + rows=source.execute(f'SELECT * FROM {table}').fetchall() + self.cursor.executemany(f"INSERT INTO {table} VALUES ({','.join(['%s']*len(rows[0]))})",rows) + source.close() + def execute(self,sql): + statements=sqlglot.parse(sql,read='mysql' if self.backend=='mysql' else 'postgres') + if len(statements)!=1 or not isinstance(statements[0],exp.Select): raise ValueError('Expected one SELECT') + tree=statements[0] + if tree.find(exp.Into): raise ValueError('SELECT INTO is not permitted') + if any(isinstance(node,(exp.DML,exp.DDL)) for node in tree.walk()): + raise ValueError('Data-changing statements are not permitted inside a query') + if any(t.name not in self.tables for t in tree.find_all(exp.Table)): raise ValueError('Query references an unknown table') + for f in tree.find_all(exp.Anonymous): + if f.name.upper() not in ['YEAR','MONTH','LOWER','UPPER']: + raise ValueError('Unsupported function '+f.name) + self.cursor.execute(sql) + rows=self.cursor.fetchmany(1001) + if len(rows)>1000: raise ValueError('Result limit exceeded') + return [tuple(str(x) if not isinstance(x,(str,int,float,type(None))) else x for x in row) for row in rows] + def close(self): self.cursor.close(); self.conn.close() + + +def main(): + p=argparse.ArgumentParser(); p.add_argument('--data',required=True); p.add_argument('--predictions') + p.add_argument('--out',required=True); p.add_argument('--limit',type=int,default=0) + args=p.parse_args() + rows=[json.loads(l) for l in Path(args.data).read_text().splitlines()] + predictions={} + if args.predictions: predictions={r['id']:r for r in map(json.loads,Path(args.predictions).read_text().splitlines())} + dbs={b:NativeDB(b) for b in ['mysql','supabase']}; results=[]; seen=set(); start=time.time() + for r in rows: + if r['operation'] not in SQL_OPS: continue + if not args.predictions: + key=(r['operation'],r['backend']) + if key in seen: continue + seen.add(key) + elif r['id'] not in predictions: continue + if args.limit and len(results)>=args.limit: break + db=dbs[r['backend']] + item={'id':r['id'],'backend':r['backend'],'operation':r['operation'],'gold_executes':False,'prediction_equivalent':None} + try: + gold_key=next(k for k in ('query','sql') if k in r['target']['arguments']) + gold=r['target']['arguments'][gold_key] + prediction=None + if args.predictions: + item['prediction_equivalent']=False + from tinyquery.evaluate import check_action,parse_action + action=parse_action(predictions[r['id']]['output']) + check_action(action,r['context']) + if action.get('name')!=r['target']['name']: raise ValueError('Wrong tool') + expected_other={k:v for k,v in r['target']['arguments'].items() if k!=gold_key} + actual_other={k:v for k,v in action['arguments'].items() if k!=gold_key} + if expected_other!=actual_other: raise ValueError('Wrong non-SQL arguments') + prediction=action['arguments'][gold_key] + equal=True + for seed in [1,7]: + db.setup(r['slots'],seed); expected=db.execute(gold) + if prediction is not None: + actual=db.execute(prediction) + ordered=r['operation'] in ['sort_asc','sort_desc','top','bottom'] + if not ordered: expected=sorted(expected,key=repr); actual=sorted(actual,key=repr) + equal=equal and actual==expected + item['gold_executes']=True + if prediction is not None: item['prediction_equivalent']=equal + except Exception as exc: item['error']=str(exc)[:300] + results.append(item) + for db in dbs.values(): db.close() + summary={'seconds':time.time()-start,'cases':len(results),'gold_executes':sum(r['gold_executes'] for r in results), + 'prediction_equivalent':sum(r['prediction_equivalent'] is True for r in results) if args.predictions else None, + 'engine_versions':{'mysql':'8.0.46','postgresql':'16.15'},'results':results} + Path(args.out).write_text(json.dumps(summary,indent=2)) + print(json.dumps({k:v for k,v in summary.items() if k!='results'}),flush=True) + for r in results: + if 'error' in r: print(json.dumps(r),flush=True) + + +if __name__=='__main__': main() diff --git a/tinyquery/package_dataset.py b/tinyquery/package_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..d597aad624c93fde01a9284b295fe3db6d6ed541 --- /dev/null +++ b/tinyquery/package_dataset.py @@ -0,0 +1,54 @@ +"""Build a Hugging Face viewer table plus lossless compressed source records.""" +import argparse +import gzip +import hashlib +import json +from pathlib import Path +import shutil +import pyarrow as pa +import pyarrow.parquet as pq + + +def main(): + p=argparse.ArgumentParser(); p.add_argument('--data',required=True);p.add_argument('--out',required=True) + p.add_argument('--teacher-source'); args=p.parse_args(); source=Path(args.data); out=Path(args.out) + (out/'data').mkdir(parents=True,exist_ok=True); (out/'raw').mkdir(exist_ok=True) + counts={} + for split in ['train','validation','test','manual']+(['development'] if (source/'development.jsonl').exists() else []): + writer=None; batch=[]; count=0 + with (source/(split+'.jsonl')).open() as stream, gzip.open(out/'raw'/(split+'.jsonl.gz'),'wt',encoding='utf-8') as raw: + for line in stream: + raw.write(line); row=json.loads(line) + # Heterogeneous runtime JSON schemas remain lossless strings in Arrow. + flat={key:row.get(key,'') for key in ['id','scenario_id','language','backend','operation','question','prompt','response','provenance']} + flat['split']=split + flat['sample_weight']=float(row.get('sample_weight',1)) + for key in ['context','target','slots']: flat[key]=json.dumps(row[key],ensure_ascii=False,separators=(',',':')) + batch.append(flat); count+=1 + if len(batch)==2000: + table=pa.Table.from_pylist(batch) + if writer is None: writer=pq.ParquetWriter(out/'data'/(split+'.parquet'),table.schema,compression='zstd') + writer.write_table(table); batch=[] + if batch: + table=pa.Table.from_pylist(batch) + if writer is None: writer=pq.ParquetWriter(out/'data'/(split+'.parquet'),table.schema,compression='zstd') + writer.write_table(table) + if writer: writer.close() + counts[split]=count + (out/'provenance').mkdir(exist_ok=True) + for name in ['tokenization.json','grounding-stats.json','split-audit.json','full-audit.json']: + if (source/name).exists(): shutil.copy2(source/name,out/'provenance'/name) + if args.teacher_source: + teacher=Path(args.teacher_source) + for name in ['templates.jsonl','verified-concrete.jsonl']: + if (teacher/name).exists(): + with (teacher/name).open('rb') as src,gzip.open(out/'provenance'/(name+'.gz'),'wb') as dest: shutil.copyfileobj(src,dest) + manifest={'splits':counts,'files':{}} + for file in sorted(out.rglob('*')): + if file.is_file() and file.name!='manifest.json': + digest=hashlib.sha256(file.read_bytes()).hexdigest() + manifest['files'][str(file.relative_to(out))]={'bytes':file.stat().st_size,'sha256':digest} + (out/'manifest.json').write_text(json.dumps(manifest,indent=2));print(json.dumps(counts)) + + +if __name__=='__main__': main() diff --git a/tinyquery/prepare.py b/tinyquery/prepare.py new file mode 100644 index 0000000000000000000000000000000000000000..a9d8f21bf7c1eee53b696d7cdc7c7c8042f8684c --- /dev/null +++ b/tinyquery/prepare.py @@ -0,0 +1,66 @@ +"""Learn a byte BPE using training text only; tokenize complete supervised records.""" +import argparse +import hashlib +import json +from pathlib import Path +import numpy as np +from tokenizers import Tokenizer, models, trainers, pre_tokenizers, decoders + +SPECIAL=['<|pad|>','<|context|>','<|user|>','<|assistant|>','<|end|>'] + + +def file_sha256(path): + h=hashlib.sha256() + with Path(path).open('rb') as stream: + for block in iter(lambda:stream.read(4*1024*1024),b''):h.update(block) + return h.hexdigest() + + +def rows(path): + with open(path) as stream: + for line in stream: yield json.loads(line) + + +def main(): + p=argparse.ArgumentParser(); p.add_argument('--data',default='data/tinyquery') + p.add_argument('--vocab',type=int,default=24000); p.add_argument('--context',type=int,default=2048) + p.add_argument('--reuse-tokenizer',action='store_true') + args=p.parse_args(); base=Path(args.data) + if args.reuse_tokenizer: + tokenizer=Tokenizer.from_file(str(base/'tokenizer.json')) + else: + tokenizer=Tokenizer(models.BPE()) + tokenizer.pre_tokenizer=pre_tokenizers.ByteLevel(add_prefix_space=False) + tokenizer.decoder=decoders.ByteLevel() + trainer=trainers.BpeTrainer(vocab_size=args.vocab,min_frequency=2,special_tokens=SPECIAL, + initial_alphabet=pre_tokenizers.ByteLevel.alphabet()) + tokenizer.train_from_iterator((r['prompt']+r['response']+'<|end|>' for r in rows(base/'train.jsonl')),trainer) + tokenizer.save(str(base/'tokenizer.json')) + stats={'vocab_size':tokenizer.get_vocab_size(),'context':args.context,'special_tokens':{s:tokenizer.token_to_id(s) for s in SPECIAL}} + for split in ['train','validation','test']: + data=list(rows(base/(split+'.jsonl'))) + encoded=[]; boundaries=[]; lengths=[]; actions=[]; kept=[]; rejected=[] + for i,r in enumerate(data): + prefix=tokenizer.encode(r['prompt']).ids + suffix=tokenizer.encode(r['response']+'<|end|>').ids + ids=prefix+suffix + if len(ids)>args.context+1: + rejected.append({'id':r['id'],'tokens':len(ids)}); continue + encoded.append(ids); boundaries.append(len(prefix)-1); lengths.append(len(ids)) + actions.append({'call':0,'clarify':1,'answer':2}[r['target']['action']]); kept.append(i) + width=max(lengths) + array=np.zeros((len(encoded),width),dtype=np.uint16) + for i,ids in enumerate(encoded): array[i,:len(ids)]=ids + np.save(base/(split+'-ids.npy'),array) + np.savez(base/(split+'-meta.npz'),boundaries=np.array(boundaries,dtype=np.int32), + lengths=np.array(lengths,dtype=np.int32),actions=np.array(actions,dtype=np.int32), + kept=np.array(kept,dtype=np.int32),sample_weights=np.array([data[i].get('sample_weight',1.0) for i in kept],dtype=np.float32)) + stats[split]={'examples':len(encoded),'tokens':sum(lengths),'response_tokens':sum(l-b-1 for l,b in zip(lengths,boundaries)), + 'ids_sha256':file_sha256(base/(split+'-ids.npy')), + 'length_p50':float(np.median(lengths)),'length_p95':float(np.quantile(lengths,.95)), + 'max_length':max(lengths),'rejected':rejected} + print(split,stats[split],flush=True) + (base/'tokenization.json').write_text(json.dumps(stats,indent=2)) + + +if __name__=='__main__': main() diff --git a/tinyquery/recipes.py b/tinyquery/recipes.py new file mode 100644 index 0000000000000000000000000000000000000000..187c6cedab6a0b4a75ed95dcfead1e269775d184 --- /dev/null +++ b/tinyquery/recipes.py @@ -0,0 +1,51 @@ +"""Semantic recipes. Placeholders are identifiers/values, never teacher-written SQL.""" + +RECIPES = { + 'all': 'Show all rows from {table}.', + 'project': 'Show only {column} from {table}.', + 'eq': 'Show all rows from {table} where {category} equals {value}.', + 'project_eq': 'Show {column} from {table} where {category} is {value}.', + 'gt': 'Show rows from {table} with {numeric} greater than {number}.', + 'lt': 'Show rows from {table} with {numeric} less than {number}.', + 'gte': 'Show rows from {table} with {numeric} at least {number}.', + 'lte': 'Show rows from {table} with {numeric} at most {number}.', + 'between': 'Show rows from {table} with {numeric} between {number} and {upper}, inclusive.', + 'and': 'Show rows from {table} where {category} is {value} and {numeric} exceeds {number}.', + 'or': 'Show rows from {table} where {category} is either {value} or {other}.', + 'null': 'Show rows from {table} where {column} is null.', + 'not_null': 'Show rows from {table} where {column} is not null.', + 'count': 'How many rows are in {table}?', + 'count_eq': 'Count rows in {table} where {category} is {value}.', + 'sum': 'Calculate the total {numeric} in {table}.', + 'avg': 'Calculate the average {numeric} in {table}.', + 'max': 'What is the maximum {numeric} in {table}?', + 'min': 'What is the minimum {numeric} in {table}?', + 'distinct': 'List distinct values of {category} in {table}.', + 'sort_desc': 'Show all rows from {table} ordered by {numeric} from highest to lowest.', + 'sort_asc': 'Show all rows from {table} ordered by {numeric} from lowest to highest.', + 'top': 'Show the top {limit} rows from {table} by {numeric}, highest first.', + 'bottom': 'Show the bottom {limit} rows from {table} by {numeric}, lowest first.', + 'group_count': 'Count rows for each {category} in {table}.', + 'group_sum': 'Show total {numeric} for each {category} in {table}.', + 'having': 'Show {category} groups in {table} with more than {number} rows and their counts.', + 'date_after': 'Show rows in {table} with {date_column} on or after {date}.', + 'year': 'Show rows in {table} where the year of {date_column} is {year}.', + 'month': 'Show rows in {table} where the month of {date_column} is {month}.', + 'contains': 'Show rows in {table} whose {column} contains the literal text {value}, case-insensitively.', + 'join': 'Show {table}.{column} and {parent}.{label} by joining {table}.{foreign} to {parent}.id.', + 'join_filter': 'Show {table}.{column} for rows joined by {table}.{foreign} to {parent}.id where {parent}.{label} equals {value}.', + 'list_tables': 'List the tables in this database.', + 'describe': 'Show the columns and types of table {table}.', + 'missing_schema': 'I have not provided a database schema. Find the available tables before writing SQL for {table}.', + 'sql_error': 'The last query failed because a column was unknown. Inspect the schema of {table} before trying again.', + 'ambiguous': 'Show the best records in {table}.', + 'missing_value': 'Filter {table} by {category}, but I have not specified a value.', + 'write': 'Delete every row from {table}.', + 'weather': 'Get the weather in {city} using {unit}.', + 'search': 'Search the documentation for {query}.', + 'read_file': 'Read the file at {path}.', + 'ticket': 'Look up support ticket {ticket_id}.', + 'final': 'The database tool returned {count} rows. Tell me how many rows it returned.', +} + +LANGUAGES = ['en', 'noisy_en', 'hi', 'hinglish'] diff --git a/tinyquery/requirements-inference.txt b/tinyquery/requirements-inference.txt new file mode 100644 index 0000000000000000000000000000000000000000..577c995eca8e5a9b85f641b58916510e0aa53d2f --- /dev/null +++ b/tinyquery/requirements-inference.txt @@ -0,0 +1,8 @@ +# Tested with PyTorch 2.13.0 on CUDA and 2.14.0 on macOS. +torch>=2.13,<3 +numpy>=2.2,<3 +tokenizers==0.22.2 +safetensors>=0.8,<1 +sqlglot==30.18.0 +jsonschema>=4.26,<5 +huggingface-hub>=1.28,<2 diff --git a/tinyquery/requirements.txt b/tinyquery/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..2f1592990286f98a6874434f25aa79b54109f313 --- /dev/null +++ b/tinyquery/requirements.txt @@ -0,0 +1,11 @@ +# Training environment used on the rented CUDA instance; torch includes its CUDA runtime. +torch==2.13.0 +numpy==2.2.6 +tokenizers==0.22.2 +safetensors==0.8.0 +sqlglot==30.18.0 +jsonschema==4.26.0 +huggingface-hub==1.28.0 +# Only needed for native database evaluation: +psycopg[binary]==3.3.5 +pymysql==1.2.0 diff --git a/tinyquery/rescore.py b/tinyquery/rescore.py new file mode 100644 index 0000000000000000000000000000000000000000..635cb0a9aeb84f5b5640ecbae7472544d7821aa0 --- /dev/null +++ b/tinyquery/rescore.py @@ -0,0 +1,28 @@ +"""Apply the current evaluator to saved raw outputs without generating new answers.""" +import argparse +from collections import defaultdict +import json +from pathlib import Path +from tinyquery.evaluate import score,aggregate + + +def rescore(data,predictions): + rows={r['id']:r for r in map(json.loads,Path(data).read_text().splitlines())};path=Path(predictions) + entries=[json.loads(l) for l in path.read_text().splitlines()];groups=defaultdict(list);changed=0 + for entry in entries: + row=rows[entry['id']];metrics=score(row,entry['output']);changed+=metrics['success']!=entry['metrics']['success'];entry['metrics']=metrics + for field in ['language','backend','operation']:groups[field+':'+row[field]].append(metrics) + tmp=path.with_suffix('.tmp');tmp.write_text(''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in entries));tmp.replace(path) + summary_path=path.with_suffix('.summary.json');summary=json.loads(summary_path.read_text()) + summary['metrics']=aggregate([r['metrics'] for r in entries]);summary['groups']={k:aggregate(v) for k,v in groups.items()} + summary['sql_metric']='Compile against the supplied schema, then compare results on two generated SQLite fixtures after dialect adaptation; not native MySQL/PostgreSQL execution.' + summary['rescoring']='Same saved raw outputs, with explicit supplied-schema compilation; no generation or output repair.' + summary_path.write_text(json.dumps(summary,indent=2));return {'predictions':str(path),'changed_success_labels':changed,'metrics':summary['metrics']} + + +def main(): + p=argparse.ArgumentParser();p.add_argument('--data',required=True);p.add_argument('--predictions',required=True) + args=p.parse_args();print(json.dumps(rescore(args.data,args.predictions))) + + +if __name__=='__main__':main() diff --git a/tinyquery/scope_data.py b/tinyquery/scope_data.py new file mode 100644 index 0000000000000000000000000000000000000000..c55d0cfb38e8c6b4937398fd41f87ada19429280 --- /dev/null +++ b/tinyquery/scope_data.py @@ -0,0 +1,56 @@ +"""Teach exact project-scope and discovery-table copying across identifier formats.""" +import argparse +import hashlib +import json +from pathlib import Path +import random +import re +import shutil +import string +from tokenizers import Tokenizer +from sqlglot import Tokenizer as SQLTokenizer +from tinyquery.data import compact,ddls,serialize + + +def main(): + p=argparse.ArgumentParser();p.add_argument('--source',default='data/tinyquery-v4') + p.add_argument('--seed-source',default='data/tinyquery');p.add_argument('--out',default='data/tinyquery-v5') + args=p.parse_args();source=Path(args.source);seeds=Path(args.seed_source);out=Path(args.out);out.mkdir(parents=True,exist_ok=True) + for name in ['validation.jsonl','test.jsonl','manual.jsonl','tokenizer.json']:shutil.copy2(source/name,out/name) + tokenizer=Tokenizer.from_file(str(source/'tokenizer.json')) + chunks=sorted({tokenizer.decode([i]) for i in range(tokenizer.get_vocab_size()) if re.fullmatch('[a-z]{1,8}',tokenizer.decode([i]))}) + rng=random.Random(99335);counts={'replay':0,'scope':0,'discovery':0};seen=set() + with (out/'train.jsonl').open('w') as stream: + def emit(row,kind): + key=hashlib.sha256(row['prompt'].encode()).hexdigest() + if key in seen:return + seen.add(key);counts[kind]+=1;stream.write(json.dumps(row,ensure_ascii=False)+'\n') + for line in (source/'train.jsonl').open():emit(json.loads(line),'replay') + for line in (seeds/'train.jsonl').open(): + row=json.loads(line) + if row['target']['action']!='call':continue + a=row['target']['arguments'];kind=None + if 'project_id' in a: + style=rng.randrange(3) + project=(''.join(rng.choices(string.ascii_lowercase,k=20)) if style==0 else + ''.join(rng.choices(chunks,k=4)) if style==1 else + '-'.join(''.join(rng.choices(string.hexdigits[:16],k=k)) for k in [8,4,4,4,12])) + row['context']['project_id']=project;a['project_id']=project;kind='scope' + if 'table' in a: + old=a['table'] + while True: + new=''.join(rng.choices(chunks,k=4)) + if len(new)>=6 and new.upper() not in SQLTokenizer.KEYWORDS:break + row['question']=re.sub(r'(?900: + rejected.append({'language':lang,'text':text,'reason':'slot_or_length'}) + continue + if lang == 'hi' and not re.search('[\u0900-\u097f]', text): + rejected.append({'language':lang,'text':text,'reason':'missing_devanagari'}) + continue + if text not in accepted[lang]: accepted[lang].append(text) + return {'op':op,'source':source,'templates':accepted,'rejected':rejected, + 'teacher':payload['model'],'revision':'017b9c7af6b5689d5dd426a76e0bc077eb5ca20a', + 'request':payload,'raw_response':raw,'usage':result.get('usage'), + 'elapsed_seconds':time.time()-started,'finish_reason':result['choices'][0]['finish_reason']} + + +def main(): + parser=argparse.ArgumentParser() + parser.add_argument('--out',default='data/tinyquery/templates.jsonl') + parser.add_argument('--base',default='http://127.0.0.1:18000/v1') + parser.add_argument('--count',type=int,default=12) + parser.add_argument('--workers',type=int,default=16) + args=parser.parse_args() + path=Path(args.out); path.parent.mkdir(parents=True,exist_ok=True) + done=set() + if path.exists(): + done={json.loads(line)['op'] for line in path.read_text().splitlines()} + started=time.time() + with path.open('a') as output, concurrent.futures.ThreadPoolExecutor(args.workers) as pool: + jobs={pool.submit(generate,op,900+i,args.base,args.count):op + for i,op in enumerate(RECIPES) if op not in done} + for future in concurrent.futures.as_completed(jobs): + op=jobs[future] + try: + result=future.result() + output.write(json.dumps(result,ensure_ascii=False)+'\n'); output.flush() + print(json.dumps({'op':op,'accepted':{k:len(v) for k,v in result['templates'].items()}, + 'seconds':round(time.time()-started,1)}),flush=True) + except Exception as exc: + print(json.dumps({'op':op,'error':str(exc)}),flush=True) + + +if __name__ == '__main__': main() diff --git a/tinyquery/test_adapter.py b/tinyquery/test_adapter.py new file mode 100644 index 0000000000000000000000000000000000000000..455b86b8da31c4cda0a3c4bdf3dd410f902b5ebf --- /dev/null +++ b/tinyquery/test_adapter.py @@ -0,0 +1,48 @@ +"""The MCP serializer must not emit a call to a different project.""" +import unittest +from tinyquery.evaluate import mcp_request,parse_action,score + + +class AdapterChecks(unittest.TestCase): + def test_execution_match_cannot_use_a_column_missing_from_context(self): + import json + from tinyquery.data import validate_sql + slots={'table':'items','parent':'items_groups','column':'name','numeric':'amount','category':'city', + 'date_column':'created_at','foreign':'group_id','label':'label','number':500,'upper':1000, + 'value':'Delhi','other':'Mumbai','date':'2025-01-15','year':2025,'month':1} + gold='SELECT name FROM items;';bad='SELECT name FROM items WHERE amount IS NULL OR amount IS NOT NULL;' + self.assertEqual(validate_sql(gold,'mysql',slots),validate_sql(bad,'mysql',slots)) + context={'schema':['CREATE TABLE items (id INTEGER, name TEXT);'],'tools':[{'name':'query','inputSchema':{ + 'type':'object','properties':{'query':{'type':'string'}},'required':['query']}}]} + row={'backend':'mysql','operation':'project','slots':slots,'context':context, + 'target':{'action':'call','name':'query','arguments':{'query':gold}}} + text=json.dumps({'action':'call','name':'query','arguments':{'query':bad}}) + self.assertFalse(score(row,text)['success']) + + def test_unknown_table_or_write_is_not_serialized(self): + context={'backend':'mysql','schema':['CREATE TABLE customers (id INTEGER);'], + 'tools':[{'name':'query','description':'Execute a read-only MySQL SELECT query.', + 'inputSchema':{'type':'object','properties':{'sql':{'type':'string'}},'required':['sql']}}]} + action={'action':'call','name':'query','arguments':{'sql':'SELECT * FROM imaginary;'}} + with self.assertRaisesRegex(ValueError,'absent'):mcp_request(action,context) + action['arguments']['sql']='DELETE FROM customers;' + with self.assertRaisesRegex(ValueError,'read-only'):mcp_request(action,context) + action['arguments']['sql']='SELECT * FROM customers;' + self.assertEqual(mcp_request(action,context)['method'],'tools/call') + + def test_duplicate_argument_keys_are_rejected(self): + with self.assertRaisesRegex(ValueError,'Duplicate JSON key'): + parse_action('{"action":"call","arguments":{"project_id":"a","project_id":"b"}}') + + def test_wrong_project_is_rejected_without_repair(self): + context={'project_id':'demo_a','tools':[{'name':'execute_sql','inputSchema':{ + 'type':'object','properties':{'project_id':{'type':'string'},'query':{'type':'string'}}, + 'required':['project_id','query'],'additionalProperties':False}}]} + action={'action':'call','name':'execute_sql','arguments':{'project_id':'demo_b','query':'SELECT 1;'}} + with self.assertRaisesRegex(ValueError,'project scope'):mcp_request(action,context) + self.assertEqual(action['arguments']['project_id'],'demo_b') + action['arguments']['project_id']='demo_a' + self.assertEqual(mcp_request(action,context)['params']['arguments']['project_id'],'demo_a') + + +if __name__=='__main__':unittest.main() diff --git a/tinyquery/test_model.py b/tinyquery/test_model.py new file mode 100644 index 0000000000000000000000000000000000000000..2aaf8d876727ad5adfea4765a96fbae76f1af40b --- /dev/null +++ b/tinyquery/test_model.py @@ -0,0 +1,39 @@ +"""Regression checks for causal decoding and the learned source-copy distribution.""" +import unittest +import torch +from tinyquery.model import Config,TinyQuery + + +class ModelChecks(unittest.TestCase): + def setUp(self): + torch.set_num_threads(2); torch.manual_seed(51) + self.model=TinyQuery(Config(vocab_size=64,width=64,layers=2,heads=4,kv_heads=2,hidden=128,context=64,copy_dim=32)).eval() + + def test_causality_and_cache(self): + x=torch.randint(5,64,(2,15));x[:,7]=3 + original=self.model(x)[0]; changed=x.clone();changed[:,9:]=torch.randint(5,64,(2,6)) + self.assertTrue(torch.allclose(original[:,:9],self.model(changed)[0][:,:9],atol=1e-6)) + past=None; parts=[] + for i in range(x.shape[1]): + logits,past,_=self.model(x[:,i:i+1],past=past,use_cache=True,last_only=True);parts.append(logits) + self.assertLess(float((original-torch.cat(parts,dim=1)).abs().max().detach()),2e-5) + self.assertTrue(torch.allclose(original.exp().sum(-1),torch.ones_like(original[:,:,0]),atol=1e-6)) + + def test_copy_cannot_recycle_its_response(self): + with torch.no_grad(): + self.model.copy_gate.weight.zero_();self.model.copy_gate.bias.fill_(-30) + self.model.copy_query.weight.zero_();self.model.copy_key.weight.zero_() + probabilities=self.model(torch.tensor([[1,5,8,3,20,20]]),last_only=True)[0].exp()[0,0] + self.assertLess(float(probabilities[20]),1e-8) + for token in [1,5,8]:self.assertAlmostEqual(float(probabilities[token]),1/3,places=6) + + def test_copy_gradient(self): + x=torch.randint(5,64,(2,15));x[:,7]=3 + y=torch.roll(x,shifts=-1,dims=1);mask=torch.arange(15)[None,:].expand(2,-1)>=7 + loss,_=self.model(x,y,mask,torch.tensor([7,7]),torch.tensor([0,0])) + loss.backward() + self.assertTrue(torch.isfinite(loss)) + self.assertGreater(float(self.model.copy_query.weight.grad.norm()),0) + + +if __name__=='__main__':unittest.main() diff --git a/tinyquery/train.py b/tinyquery/train.py new file mode 100644 index 0000000000000000000000000000000000000000..a466c6a7e136646b5786f8b14d5deb019630f536 --- /dev/null +++ b/tinyquery/train.py @@ -0,0 +1,172 @@ +"""Deadline-aware single-GPU trainer. One resumable checkpoint, atomic replacement.""" +import argparse +import contextlib +import json +import math +import os +import random +import shutil +import signal +import time +from datetime import datetime,timezone +from pathlib import Path +import numpy as np +import torch +from tinyquery.model import Config,TinyQuery + + +class Data: + def __init__(self,base,split,device): + self.ids=np.load(base/(split+'-ids.npy'),mmap_mode='r') + m=np.load(base/(split+'-meta.npz')) + self.lengths=m['lengths']; self.boundaries=m['boundaries']; self.actions=m['actions']; self.device=device + self.sample_weights=m['sample_weights'] if 'sample_weights' in m else np.ones(len(self.lengths)) + assert np.isfinite(self.sample_weights).all() and (self.sample_weights>0).all() + self.buckets={} + # Narrower length groups reduce padding while preserving each row's sampling weight. + for limit in [128,192,256,320,384,448,512,640,768,1024,1536,2049]: + low=max([x for x in self.buckets]+[0]) + indices=np.flatnonzero((self.lengths>low)&(self.lengths<=limit)) + if len(indices): self.buckets[limit]=indices + self.keys=list(self.buckets); self.probs=np.array([self.sample_weights[self.buckets[k]].sum() for k in self.keys],dtype=float) + self.probs/=self.probs.sum() + self.within={k:self.sample_weights[v].astype(float)/self.sample_weights[v].sum() for k,v in self.buckets.items()} + def batch(self,size,rng): + limit=rng.choice(self.keys,p=self.probs); indices=rng.choice(self.buckets[limit],size=size,p=self.within[limit]) + return self.from_indices(indices) + def from_indices(self,indices): + width=min(self.ids.shape[1],int(self.lengths[indices].max())) + raw=torch.tensor(np.array(self.ids[indices,:width],dtype=np.int64),device=self.device) + lengths=torch.tensor(self.lengths[indices],device=self.device) + boundaries=torch.tensor(self.boundaries[indices],device=self.device,dtype=torch.long) + actions=torch.tensor(self.actions[indices],device=self.device,dtype=torch.long) + x=raw[:,:-1]; y=raw[:,1:].clone() + positions=torch.arange(y.shape[1],device=self.device)[None,:] + y.masked_fill_(positions>=lengths[:,None]-1,-100) + weights=positions>=boundaries[:,None] + return x,y,weights,boundaries,actions,int((lengths-1).sum()),int((lengths-boundaries-1).sum()) + + +def main(): + p=argparse.ArgumentParser(); p.add_argument('--data',required=True); p.add_argument('--out',required=True) + p.add_argument('--minutes',type=float,default=150); p.add_argument('--deadline',help='Optional hard UTC deadline in ISO 8601 format') + p.add_argument('--batch',type=int,default=16); p.add_argument('--accum',type=int,default=2) + p.add_argument('--lr',type=float,default=0.0006); p.add_argument('--resume',action='store_true') + p.add_argument('--copy-dim',type=int,default=0); p.add_argument('--init-from') + p.add_argument('--steps',type=int,default=0); p.add_argument('--compile',action='store_true') + p.add_argument('--save-seconds',type=float,default=120) + p.add_argument('--width',type=int,default=1024); p.add_argument('--layers',type=int,default=12) + p.add_argument('--heads',type=int,default=16); p.add_argument('--kv-heads',type=int,default=4) + p.add_argument('--hidden',type=int,default=2816); p.add_argument('--prompt-weight',type=float,default=.15) + args=p.parse_args(); base=Path(args.data); out=Path(args.out); out.mkdir(parents=True,exist_ok=True) + torch.manual_seed(20260910); np.random.seed(20260910); random.seed(20260910) + device='cuda' if torch.cuda.is_available() else ('mps' if torch.backends.mps.is_available() else 'cpu') + if device=='cuda': torch.backends.cuda.matmul.allow_tf32=True + torch.set_num_threads(8) + token_info=json.loads((base/'tokenization.json').read_text()) + config=Config(vocab_size=token_info['vocab_size'],width=args.width,layers=args.layers,heads=args.heads, + kv_heads=args.kv_heads,hidden=args.hidden,context=token_info['context'],copy_dim=args.copy_dim) + model=TinyQuery(config).to(device) + params=sum(p.numel() for p in model.parameters()); assert params<500_000_000 + optimizer=torch.optim.AdamW(model.parameters(),lr=args.lr,betas=(.9,.95),weight_decay=.1,fused=device=='cuda') + step=0; processed=0; response_tokens=0; prior_seconds=0 + rng=np.random.default_rng(42) + if args.resume: + checkpoint=torch.load(out/'last.pt',map_location=device,weights_only=False) + assert Config(**checkpoint['config']).to_dict()==config.to_dict() + model.load_state_dict(checkpoint['model']); optimizer.load_state_dict(checkpoint['optimizer']) + step=checkpoint['step']; processed=checkpoint['processed_tokens']; response_tokens=checkpoint['response_tokens'] + prior_seconds=checkpoint.get('training_seconds',0); rng.bit_generator.state=checkpoint['rng'] + del checkpoint + elif args.init_from: + checkpoint=torch.load(args.init_from,map_location=device,weights_only=False) + previous=Config(**checkpoint['config']).to_dict(); current=config.to_dict() + assert {k:v for k,v in previous.items() if k!='copy_dim'}=={k:v for k,v in current.items() if k!='copy_dim'} + missing,unexpected=model.load_state_dict(checkpoint['model'],strict=False) + assert not unexpected and all(name.startswith('copy_') for name in missing) + processed=checkpoint['processed_tokens']; response_tokens=checkpoint['response_tokens'] + prior_seconds=checkpoint.get('training_seconds',0) + print(json.dumps({'event':'initialize_from_own_checkpoint','parent_step':checkpoint['step'],'new_parameters':missing}),flush=True) + del checkpoint + train=Data(base,'train',device); val=Data(base,'validation',device) + (out/'config.json').write_text(json.dumps(config.to_dict(),indent=2)) + (out/'run-args.json').write_text(json.dumps(vars(args),indent=2)) + runner=torch.compile(model,dynamic=True) if args.compile else model + start=time.time(); stop=min(start+args.minutes*60,datetime.fromisoformat(args.deadline).timestamp() if args.deadline else float('inf')) + if stop<=start: raise ValueError('Training deadline has already passed') + last_save=start; last_log=start; initial_step=step; initial_tokens=processed + stopping=False + def request_stop(signum,frame): + nonlocal stopping + stopping=True + signal.signal(signal.SIGTERM,request_stop) + signal.signal(signal.SIGINT,request_stop) + best_path=out/'best-info.json' + best_loss=json.loads(best_path.read_text())['response_loss'] if best_path.exists() else float('inf') + autocast=lambda: torch.autocast('cuda',dtype=torch.bfloat16) if device=='cuda' else contextlib.nullcontext() + def save(final=False): + expected=params*12+200_000_000 + free=shutil.disk_usage(out).free + if free=best_loss: return + from safetensors.torch import save_file + state={k:v.detach().cpu().to(torch.bfloat16).contiguous() for k,v in model.state_dict().items()} + temp=out/'best.tmp.safetensors'; save_file(state,str(temp),metadata={'step':str(step),'validation_response_loss':str(metrics[1]),'random_initialization':'true'}); os.replace(temp,out/'best.safetensors') + best_loss=metrics[1] + best_path.write_text(json.dumps({'step':step,'response_loss':best_loss,'validation':metrics},indent=2)) + print(json.dumps({'event':'best','step':step,'response_loss':best_loss}),flush=True) + print(json.dumps({'event':'start','parameters':params,'device':device,'config':config.to_dict(), + 'deadline':datetime.fromtimestamp(stop,timezone.utc).isoformat(),'validation':validate()}),flush=True) + with (out/'metrics.jsonl').open('a') as log: + model.train() + while time.time()=args.steps: break + progress=min(1,(time.time()-start)/(stop-start)) + warm=min(1,(step+1)/100) + lr=args.lr*warm*(.1+.9*.5*(1+math.cos(math.pi*progress))) + for group in optimizer.param_groups: group['lr']=lr + optimizer.zero_grad(set_to_none=True); sums=torch.zeros(3,device=device) + for _ in range(args.accum): + x,y,w,b,a,nt,nr=train.batch(args.batch,rng) + with autocast(): loss,parts=runner(x,y,w,b,a,prompt_weight=args.prompt_weight) + (loss/args.accum).backward(); sums+=parts + processed+=nt; response_tokens+=nr + norm=torch.nn.utils.clip_grad_norm_(model.parameters(),1.0) + if not torch.isfinite(norm): raise RuntimeError('Non-finite gradient; refusing corrupt checkpoint') + optimizer.step(); step+=1 + now=time.time() + if now-last_log>20 or step<=3: + entry={'step':step,'elapsed_seconds':prior_seconds+now-start,'processed_tokens':processed, + 'response_tokens':response_tokens,'tokens_per_second':(processed-initial_tokens)/(now-start), + 'loss':(sums/args.accum).tolist(),'lr':lr,'gradient_norm':float(norm)} + log.write(json.dumps(entry)+'\n'); log.flush(); print(json.dumps(entry),flush=True); last_log=now + if now-last_save>args.save_seconds: + val_metrics=validate(); save_best(val_metrics); save(); last_save=time.time() + final_val=validate(); save_best(final_val); save(final=True) + summary={'parameters':params,'step':step,'processed_tokens':processed,'response_tokens':response_tokens, + 'training_seconds':prior_seconds+time.time()-start,'validation':final_val, + 'random_initialization':True,'device':device} + (out/'summary.json').write_text(json.dumps(summary,indent=2)); print(json.dumps(summary),flush=True) + + +if __name__=='__main__': main() diff --git a/tinyquery/verify_concrete.py b/tinyquery/verify_concrete.py new file mode 100644 index 0000000000000000000000000000000000000000..08311734745b1082f1d04a8ed069198d3c98a958 --- /dev/null +++ b/tinyquery/verify_concrete.py @@ -0,0 +1,65 @@ +"""Round-trip direct paraphrases to SQL without exposing reference SQL to the verifier.""" +import argparse +import concurrent.futures +import json +import time +import urllib.request +from pathlib import Path +from tinyquery.data import compact,validate_sql + + +def normalized(rows,ordered): + return rows if ordered else sorted(rows,key=lambda row:repr(row)) + + +def verify(item,base): + records=item['accepted'] + if not records: return {'scenario_id':item['scenario_id'],'accepted':[],'rejected':[]} + sample=records[0] + payload={'model':'Qwen/Qwen3.8-27B-FP8','messages':[{'role':'user','content': + 'Translate each request to one read-only '+('PostgreSQL' if sample['backend']=='supabase' else 'MySQL')+ + ' SELECT query using only this schema. Return a JSON object mapping each language key to its SQL string. ' + 'If a request is ambiguous or impossible, use an empty string. Do not add constraints. '+ + compact({'schema':sample['context']['schema'],'requests':{r['language']:r['question'] for r in records}})}], + 'max_tokens':700,'temperature':0,'response_format':{'type':'json_object'}, + 'chat_template_kwargs':{'enable_thinking':False}} + request=urllib.request.Request(base+'/chat/completions',data=compact(payload).encode(),headers={'Content-Type':'application/json'}) + with urllib.request.urlopen(request,timeout=180) as response: raw=json.load(response) + answers=json.loads(raw['choices'][0]['message']['content']) + gold=next(v for k,v in sample['target']['arguments'].items() if k in ['query','sql']) + reference=validate_sql(gold,sample['backend'],sample['slots']) + ordered=sample['operation'] in ['top','bottom','sort_asc','sort_desc'] + accepted=[]; rejected=[] + for record in records: + try: + sql=answers[record['language']] + results=validate_sql(sql,record['backend'],record['slots']) + if any(normalized(a,ordered)!=normalized(b,ordered) for a,b in zip(reference,results)): + raise ValueError('SQL results differ on fixtures') + record['roundtrip_sql']=sql + record['provenance']+='; teacher round-trip SQL equivalent on two generated SQLite fixtures' + accepted.append(record) + except Exception as exc: + rejected.append({'id':record['id'],'question':record['question'],'reason':str(exc),'sql':answers.get(record['language'])}) + return {'scenario_id':item['scenario_id'],'accepted':accepted,'rejected':rejected,'verification_request':payload,'verification_response':raw} + + +def main(): + p=argparse.ArgumentParser(); p.add_argument('--input',required=True); p.add_argument('--out',required=True) + p.add_argument('--workers',type=int,default=24); p.add_argument('--base',default='http://127.0.0.1:18000/v1') + args=p.parse_args(); output=Path(args.out); done=set() + if output.exists(): done={json.loads(l)['scenario_id'] for l in output.read_text().splitlines()} + items=[json.loads(l) for l in Path(args.input).read_text().splitlines()] + start=time.time(); completed=0; accepted=0; rejected=0 + with output.open('a') as stream,concurrent.futures.ThreadPoolExecutor(args.workers) as pool: + jobs={pool.submit(verify,r,args.base):r['scenario_id'] for r in items if r['scenario_id'] not in done} + for future in concurrent.futures.as_completed(jobs): + completed+=1 + try: + result=future.result(); accepted+=len(result['accepted']); rejected+=len(result['rejected']) + stream.write(compact(result)+'\n'); stream.flush() + except Exception as exc: print(compact({'error':str(exc),'scenario_id':jobs[future]}),flush=True) + if completed%25==0: print(compact({'completed':completed,'accepted':accepted,'rejected':rejected,'seconds':time.time()-start}),flush=True) + + +if __name__=='__main__': main() diff --git a/tokenization.json b/tokenization.json new file mode 100644 index 0000000000000000000000000000000000000000..b749d9ea662d1ea6ef277e8e6d37bc22db89ce8f --- /dev/null +++ b/tokenization.json @@ -0,0 +1,41 @@ +{ + "vocab_size": 4082, + "context": 2048, + "special_tokens": { + "<|pad|>": 0, + "<|context|>": 1, + "<|user|>": 2, + "<|assistant|>": 3, + "<|end|>": 4 + }, + "train": { + "examples": 347376, + "tokens": 100224712, + "response_tokens": 13163249, + "length_p50": 278.0, + "length_p95": 389.0, + "max_length": 945, + "rejected": [], + "ids_sha256": "365f03740dee120064b98b530f4b67e987f1cf192435281bfd9d3f32f4271f75" + }, + "validation": { + "examples": 1200, + "tokens": 336514, + "response_tokens": 42286, + "length_p50": 276.0, + "length_p95": 374.0, + "max_length": 406, + "rejected": [], + "ids_sha256": "7bdf2a15c82cd7459e2410cd3fd7cf70410392780e9979228b1edaec1914e4bf" + }, + "test": { + "examples": 1196, + "tokens": 339535, + "response_tokens": 43042, + "length_p50": 279.0, + "length_p95": 388.0, + "max_length": 426, + "rejected": [], + "ids_sha256": "1c668ade4e6eff7d1e62697b39419ca59befc572f6961802cad0570e15166d41" + } +} \ No newline at end of file diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..aa3079a933168a5b8218a278646261d8fab48fbd --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,19446 @@ +{ + "version": "1.0", + "truncation": null, + "padding": null, + "added_tokens": [ + { + "id": 0, + "content": "<|pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1, + "content": "<|context|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2, + "content": "<|user|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3, + "content": "<|assistant|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4, + "content": "<|end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + } + ], + "normalizer": null, + "pre_tokenizer": { + "type": "ByteLevel", + "add_prefix_space": false, + "trim_offsets": true, + "use_regex": true + }, + "post_processor": null, + "decoder": { + "type": "ByteLevel", + "add_prefix_space": true, + "trim_offsets": true, + "use_regex": true + }, + "model": { + "type": "BPE", + "dropout": null, + "unk_token": null, + "continuing_subword_prefix": null, + "end_of_word_suffix": null, + "fuse_unk": false, + "byte_fallback": false, + "ignore_merges": false, + "vocab": { + "<|pad|>": 0, + "<|context|>": 1, + "<|user|>": 2, + "<|assistant|>": 3, + "<|end|>": 4, + "!": 5, + "\"": 6, + "#": 7, + "$": 8, + "%": 9, + "&": 10, + "'": 11, + "(": 12, + ")": 13, + "*": 14, + "+": 15, + ",": 16, + "-": 17, + ".": 18, + "/": 19, + "0": 20, + "1": 21, + "2": 22, + "3": 23, + "4": 24, + "5": 25, + "6": 26, + "7": 27, + "8": 28, + "9": 29, + ":": 30, + ";": 31, + "<": 32, + "=": 33, + ">": 34, + "?": 35, + "@": 36, + "A": 37, + "B": 38, + "C": 39, + "D": 40, + "E": 41, + "F": 42, + "G": 43, + "H": 44, + "I": 45, + "J": 46, + "K": 47, + "L": 48, + "M": 49, + "N": 50, + "O": 51, + "P": 52, + "Q": 53, + "R": 54, + "S": 55, + "T": 56, + "U": 57, + "V": 58, + "W": 59, + "X": 60, + "Y": 61, + "Z": 62, + "[": 63, + "\\": 64, + "]": 65, + "^": 66, + "_": 67, + "`": 68, + "a": 69, + "b": 70, + "c": 71, + "d": 72, + "e": 73, + "f": 74, + "g": 75, + "h": 76, + "i": 77, + "j": 78, + "k": 79, + "l": 80, + "m": 81, + "n": 82, + "o": 83, + "p": 84, + "q": 85, + "r": 86, + "s": 87, + "t": 88, + "u": 89, + "v": 90, + "w": 91, + "x": 92, + "y": 93, + "z": 94, + "{": 95, + "|": 96, + "}": 97, + "~": 98, + "¡": 99, + "¢": 100, + "£": 101, + "¤": 102, + "¥": 103, + "¦": 104, + "§": 105, + "¨": 106, + "©": 107, + "ª": 108, + "«": 109, + "¬": 110, + "®": 111, + "¯": 112, + "°": 113, + "±": 114, + "²": 115, + "³": 116, + "´": 117, + "µ": 118, + "¶": 119, + "·": 120, + "¸": 121, + "¹": 122, + "º": 123, + "»": 124, + "¼": 125, + "½": 126, + "¾": 127, + "¿": 128, + "À": 129, + "Á": 130, + "Â": 131, + "Ã": 132, + "Ä": 133, + "Å": 134, + "Æ": 135, + "Ç": 136, + "È": 137, + "É": 138, + "Ê": 139, + "Ë": 140, + "Ì": 141, + "Í": 142, + "Î": 143, + "Ï": 144, + "Ð": 145, + "Ñ": 146, + "Ò": 147, + "Ó": 148, + "Ô": 149, + "Õ": 150, + "Ö": 151, + "×": 152, + "Ø": 153, + "Ù": 154, + "Ú": 155, + "Û": 156, + "Ü": 157, + "Ý": 158, + "Þ": 159, + "ß": 160, + "à": 161, + "á": 162, + "â": 163, + "ã": 164, + "ä": 165, + "å": 166, + "æ": 167, + "ç": 168, + "è": 169, + "é": 170, + "ê": 171, + "ë": 172, + "ì": 173, + "í": 174, + "î": 175, + "ï": 176, + "ð": 177, + "ñ": 178, + "ò": 179, + "ó": 180, + "ô": 181, + "õ": 182, + "ö": 183, + "÷": 184, + "ø": 185, + "ù": 186, + "ú": 187, + "û": 188, + "ü": 189, + "ý": 190, + "þ": 191, + "ÿ": 192, + "Ā": 193, + "ā": 194, + "Ă": 195, + "ă": 196, + "Ą": 197, + "ą": 198, + "Ć": 199, + "ć": 200, + "Ĉ": 201, + "ĉ": 202, + "Ċ": 203, + "ċ": 204, + "Č": 205, + "č": 206, + "Ď": 207, + "ď": 208, + "Đ": 209, + "đ": 210, + "Ē": 211, + "ē": 212, + "Ĕ": 213, + "ĕ": 214, + "Ė": 215, + "ė": 216, + "Ę": 217, + "ę": 218, + "Ě": 219, + "ě": 220, + "Ĝ": 221, + "ĝ": 222, + "Ğ": 223, + "ğ": 224, + "Ġ": 225, + "ġ": 226, + "Ģ": 227, + "ģ": 228, + "Ĥ": 229, + "ĥ": 230, + "Ħ": 231, + "ħ": 232, + "Ĩ": 233, + "ĩ": 234, + "Ī": 235, + "ī": 236, + "Ĭ": 237, + "ĭ": 238, + "Į": 239, + "į": 240, + "İ": 241, + "ı": 242, + "IJ": 243, + "ij": 244, + "Ĵ": 245, + "ĵ": 246, + "Ķ": 247, + "ķ": 248, + "ĸ": 249, + "Ĺ": 250, + "ĺ": 251, + "Ļ": 252, + "ļ": 253, + "Ľ": 254, + "ľ": 255, + "Ŀ": 256, + "ŀ": 257, + "Ł": 258, + "ł": 259, + "Ń": 260, + "\":": 261, + ",\"": 262, + "\":\"": 263, + "{\"": 264, + "ti": 265, + "\",\"": 266, + "es": 267, + "pe": 268, + "ro": 269, + "on": 270, + "re": 271, + "\":{\"": 272, + "in": 273, + "TE": 274, + "se": 275, + "ta": 276, + "ty": 277, + "tion": 278, + "qu": 279, + "he": 280, + "}}": 281, + "à¤": 282, + "id": 283, + "me": 284, + "tab": 285, + "ec": 286, + "type": 287, + "al": 288, + "st": 289, + "pro": 290, + "te": 291, + "ma": 292, + "per": 293, + "ties": 294, + "perties": 295, + "Ġa": 296, + "RE": 297, + "ad": 298, + "ect": 299, + "ject": 300, + "\":[": 301, + "che": 302, + "chema": 303, + "nt": 304, + "],\"": 305, + "ol": 306, + "er": 307, + "na": 308, + "name": 309, + "ing": 310, + "ri": 311, + "tabl": 312, + "<|": 313, + "|>": 314, + "Ġc": 315, + "ery": 316, + "query": 317, + "cri": 318, + "LE": 319, + "is": 320, + "ase": 321, + "red": 322, + "qui": 323, + "requi": 324, + "required": 325, + "up": 326, + "da": 327, + "nd": 328, + "\"}}": 329, + "des": 330, + "descri": 331, + "XT": 332, + "REA": 333, + "ption": 334, + "ĠTE": 335, + "ĠTEXT": 336, + "ring": 337, + "string": 338, + "or": 339, + "ĠI": 340, + "ER": 341, + "tabase": 342, + "database": 343, + "\":[\"": 344, + "ĠIN": 345, + "description": 346, + "GER": 347, + "TEGER": 348, + "ĠINTEGER": 349, + "gro": 350, + "group": 351, + "\"],\"": 352, + "pu": 353, + "di": 354, + ".\",\"": 355, + "put": 356, + "nts": 357, + "ob": 358, + "fal": 359, + "Pro": 360, + "dition": 361, + "Schema": 362, + "input": 363, + "alPro": 364, + "properties": 365, + "addition": 366, + "object": 367, + "false": 368, + "inputSchema": 369, + "alProperties": 370, + "additionalProperties": 371, + "Ġt": 372, + "ar": 373, + "ql": 374, + "sql": 375, + "de": 376, + "ns": 377, + "à¥": 378, + "Ġà¤": 379, + "ĠP": 380, + "ool": 381, + "project": 382, + ");": 383, + "ute": 384, + "xec": 385, + "xecute": 386, + "ac": 387, + "Ġ(": 388, + "ĠT": 389, + "ist": 390, + "AB": 391, + "CREA": 392, + "ĠTAB": 393, + "CREATE": 394, + "ĠTABLE": 395, + "ools": 396, + "schema": 397, + "tables": 398, + "ly": 399, + "table": 400, + "ments": 401, + "Ġdatabase": 402, + "only": 403, + "end": 404, + "|>{\"": 405, + "groups": 406, + "ic": 407, + "mo": 408, + ",{\"": 409, + "}},{\"": 410, + "CT": 411, + "IM": 412, + "KE": 413, + "AR": 414, + "ELE": 415, + "ELECT": 416, + "SELECT": 417, + "for": 418, + "RIM": 419, + "ĠKE": 420, + "ĠPRIM": 421, + "ARY": 422, + "ĠKEY": 423, + "ĠPRIMARY": 424, + "\"}},\"": 425, + "um": 426, + "\"}": 427, + "ay": 428, + "ab": 429, + "Ġs": 430, + "Ġre": 431, + "Ġand": 432, + "Ġcol": 433, + "Ġcolum": 434, + "Ġin": 435, + "Ġcolumns": 436, + "at": 437, + "Ġw": 438, + "Ġthe": 439, + "ĠD": 440, + "ts": 441, + "Ġpro": 442, + "Ġac": 443, + "ERE": 444, + "ATE": 445, + "ĠREA": 446, + ");\",\"": 447, + "ĠDATE": 448, + "ĠREAL": 449, + "Ġacc": 450, + "tem": 451, + "Re": 452, + "item": 453, + "demo": 454, + "Ġis": 455, + "execute": 456, + "Ġwhe": 457, + "el": 458, + "db": 459, + "tools": 460, + "Ġquery": 461, + "ĠA": 462, + "xt": 463, + "ed": 464, + "text": 465, + "list": 466, + "Read": 467, + "schemas": 468, + "con": 469, + "Ġl": 470, + "\":[{\"": 471, + "Ġm": 472, + "Ġtables": 473, + "Ġread": 474, + "as": 475, + "QL": 476, + "SQL": 477, + "ĠSELECT": 478, + "List": 479, + "ir": 480, + "items": 481, + "vid": 482, + "ess": 483, + "Ġprovid": 484, + "bac": 485, + "back": 486, + "sing": 487, + "use": 488, + "sk": 489, + "Ġtheir": 490, + "Ġprovided": 491, + "forma": 492, + "Ġwhen": 493, + "Ġinforma": 494, + "Ġinformation": 495, + "ass": 496, + "issing": 497, + "Ġmissing": 498, + "Ġaccess": 499, + "ĠU": 500, + "tant": 501, + "Execute": 502, + "pol": 503, + "ray": 504, + "},\"": 505, + "Ġrequired": 506, + "}}],\"": 507, + "istant": 508, + "\"}}},\"": 509, + "Ġtools": 510, + "array": 511, + "action": 512, + "icy": 513, + "ĠAsk": 514, + "context": 515, + "backend": 516, + "user": 517, + "assistant": 518, + "ĠUse": 519, + "policy": 520, + "ou": 521, + ".\"}": 522, + "all": 523, + "abel": 524, + "ES": 525, + "arg": 526, + "call": 527, + "uments": 528, + "arguments": 529, + "ke": 530, + "ate": 531, + "ity": 532, + "Ġgroup": 533, + "Ġlabel": 534, + "CES": 535, + "FERE": 536, + "NCES": 537, + "ĠRE": 538, + ");\"],\"": 539, + "FERENCES": 540, + "ĠREFERENCES": 541, + "\":[],\"": 542, + "\"}}<|": 543, + "));\"],\"": 544, + "RO": 545, + "ा": 546, + "à¥ĩ": 547, + "tic": 548, + "ices": 549, + "ĠF": 550, + "ticke": 551, + "ROM": 552, + "ĠFROM": 553, + "Ġfor": 554, + "ord": 555, + "Ġb": 556, + ";\"}}<|": 557, + "Ġcity": 558, + "ost": 559, + "pl": 560, + "Ġe": 561, + "order": 562, + "reate": 563, + "my": 564, + "mysql": 565, + "reated": 566, + "Ġcreated": 567, + "sup": 568, + "abase": 569, + "supabase": 570, + "Ġp": 571, + "mou": 572, + "mount": 573, + "ĠM": 574, + "Ġamount": 575, + "\"},\"": 576, + "à¤Ĥ": 577, + "un": 578, + "Ġk": 579, + "Ġn": 580, + "an": 581, + "Ġorder": 582, + "pec": 583, + "Ġtable": 584, + "__": 585, + "Ġon": 586, + "pect": 587, + "nspect": 588, + "Ġde": 589, + "be": 590, + "describe": 591, + "pp": 592, + "le": 593, + "pes": 594, + "Ġty": 595, + "Ġtypes": 596, + "Ġà¤ķ": 597, + "hi": 598, + "Ġone": 599, + "its": 600, + "Inspect": 601, + "date": 602, + "gre": 603, + "ySQL": 604, + "ĠMySQL": 605, + "ĠPost": 606, + "greSQL": 607, + "ĠPostgreSQL": 608, + "Ġsta": 609, + "ok": 610, + "à¥į": 611, + "ook": 612, + "ooks": 613, + "Ġf": 614, + "ि": 615, + "ye": 616, + "Ġ*": 617, + "Ġau": 618, + "ot": 619, + "uc": 620, + "vis": 621, + "visits": 622, + "Ġnot": 623, + "duc": 624, + "ducts": 625, + "oin": 626, + "र": 627, + "ĠW": 628, + "ce": 629, + "mpl": 630, + "tickets": 631, + "oye": 632, + "mploye": 633, + "mployees": 634, + "ub": 635, + "ptions": 636, + "ubs": 637, + "ubscri": 638, + "vices": 639, + "ayments": 640, + "ubscriptions": 641, + "vo": 642, + "rs": 643, + "voices": 644, + "ours": 645, + "ourses": 646, + "HERE": 647, + "ĠWHERE": 648, + "hip": 649, + "hipments": 650, + "ude": 651, + "ounts": 652, + "Ġaux": 653, + "Ġnotes": 654, + "ticket": 655, + "stude": 656, + "students": 657, + "tments": 658, + "ppoin": 659, + "ppointments": 660, + "et": 661, + "pens": 662, + "xpens": 663, + "xpenses": 664, + "it": 665, + "us": 666, + "Ġd": 667, + "à¤ķ": 668, + "row": 669, + "Ġrec": 670, + "run": 671, + "Ġbooks": 672, + "Ġvisits": 673, + "Ġस": 674, + "Ġrow": 675, + "Ġtickets": 676, + "Ġproducts": 677, + "Ġpayments": 678, + "Ġemployees": 679, + "Ġdevices": 680, + "Ġsubscriptions": 681, + "Ġcourses": 682, + "Ġinvoices": 683, + "ha": 684, + "ory": 685, + "ath": 686, + "rom": 687, + "Ġfrom": 688, + "Ġshipments": 689, + "orde": 690, + "cp": 691, + "mcp": 692, + "à¥Ģ": 693, + "à¥ĩà¤Ĥ": 694, + "Ġorders": 695, + "Ġstudents": 696, + "Ġrows": 697, + "त": 698, + "ind": 699, + "00": 700, + "Ġappointments": 701, + "orded": 702, + "Ġaccounts": 703, + "Ġcu": 704, + "Ġexpenses": 705, + "Ġrecorded": 706, + "rt": 707, + "gi": 708, + "Ġstart": 709, + "।": 710, + "न": 711, + "ch": 712, + "ispl": 713, + "isplay": 714, + "tus": 715, + "Ġkind": 716, + "art": 717, + "Ġstatus": 718, + "ment": 719, + "gion": 720, + "part": 721, + "partment": 722, + "Ġregion": 723, + "Ġdepartment": 724, + "ateg": 725, + "ategory": 726, + "Ġcategory": 727, + "get": 728, + "य": 729, + "tin": 730, + "ore": 731, + "ting": 732, + "Ġ1": 733, + "ve": 734, + "oc": 735, + "ra": 736, + "Ġ=": 737, + "Ġse": 738, + "tit": 739, + "title": 740, + "Ġme": 741, + "her": 742, + "Ġप": 743, + "rating": 744, + "core": 745, + "ĠB": 746, + "sto": 747, + "ile": 748, + "stome": 749, + "stomer": 750, + "Ġम": 751, + "Ġdisplay": 752, + "rice": 753, + "alan": 754, + "eat": 755, + "eather": 756, + "alance": 757, + "Ġtitle": 758, + "ow": 759, + "arch": 760, + "ल": 761, + "Ġscore": 762, + "Ġrating": 763, + "Ġitem": 764, + "Ġcost": 765, + "tity": 766, + "city": 767, + "quan": 768, + "quantity": 769, + "unit": 770, + "ary": 771, + "alary": 772, + "Ġह": 773, + "path": 774, + "Ġdescription": 775, + "Ġprice": 776, + "Ġha": 777, + "Ġbalance": 778, + "Ġcustomer": 779, + "Ġname": 780, + "Ġquantity": 781, + "Ġsalary": 782, + "Ġwhere": 783, + "à¥ĭ": 784, + "Ġ'": 785, + "म": 786, + "ah": 787, + "ĠBY": 788, + "à¥Ī": 789, + "à¤ģ": 790, + "Ġ2": 791, + "Ġka": 792, + "ĠO": 793, + "ह": 794, + "Ġto": 795, + "app": 796, + "\"]": 797, + "\"]}}": 798, + "NT": 799, + "ort": 800, + "à¤ı": 801, + "स": 802, + "Ġby": 803, + "Ġ5": 804, + "books": 805, + "Ġj": 806, + "products": 807, + "Ġhai": 808, + "Ġ>": 809, + "employees": 810, + "shipments": 811, + "invoices": 812, + "ocs": 813, + "ाà¤ģ": 814, + "subscriptions": 815, + "devices": 816, + "courses": 817, + "ĠL": 818, + "counts": 819, + "accounts": 820, + "appointments": 821, + "ĠC": 822, + "payments": 823, + "orders": 824, + "IN": 825, + "OU": 826, + "(*": 827, + "ĠCOU": 828, + "(*)": 829, + "ĠCOUNT": 830, + "';\"}}<|": 831, + "docs": 832, + "expenses": 833, + "Ġà¤ľ": 834, + "Ġ20": 835, + "à¤Ł": 836, + "tri": 837, + "eve": 838, + "trieve": 839, + "ren": 840, + "si": 841, + "Get": 842, + "search": 843, + "file": 844, + "read": 845, + "elsi": 846, + "elsius": 847, + "weather": 848, + "heit": 849, + "ahren": 850, + "ahrenheit": 851, + "Retrieve": 852, + "प": 853, + "Ġद": 854, + "Ġfile": 855, + "Ġticket": 856, + "en": 857, + "à¤ĸ": 858, + "celsius": 859, + "Ġweather": 860, + "fahrenheit": 861, + "how": 862, + "DER": 863, + "RDER": 864, + "ĠORDER": 865, + "ब": 866, + "port": 867, + "ĠN": 868, + "ocu": 869, + "il": 870, + "Ġव": 871, + "nta": 872, + "menta": 873, + "ocumenta": 874, + "ocumentation": 875, + "Ġsup": 876, + "Ġsupport": 877, + "label": 878, + "Ġke": 879, + "Ġdocumentation": 880, + "Ġv": 881, + "est": 882, + "ue": 883, + "rent": 884, + "Ġits": 885, + "à¥ĩà¤Ĥ।": 886, + "Ġtext": 887, + "Se": 888, + "rrent": 889, + "Ġcurrent": 890, + "Search": 891, + "ĠID": 892, + "Ġpath": 893, + "\"]}},\"": 894, + "enum": 895, + "Ġà¤ķर": 896, + "cl": 897, + "Ġmein": 898, + "Ġth": 899, + "Ġki": 900, + "Ġval": 901, + "à¥Ī।": 902, + "ari": 903, + "Ġन": 904, + "ll": 905, + "Ġkaro": 906, + "fy": 907, + "Ġy": 908, + "Ġo": 909, + "ĠG": 910, + "UP": 911, + "ROUP": 912, + "ĠGROUP": 913, + "Ġ100": 914, + "\"}<|": 915, + "à¥Ĥ": 916, + "ĠS": 917, + "Ġ202": 918, + "Ġdi": 919, + "Ġma": 920, + "Ġof": 921, + "Ġल": 922, + "lic": 923, + "à¥ģ": 924, + "?\"}<|": 925, + "estion": 926, + "question": 927, + "clari": 928, + "clarify": 929, + "blic": 930, + "public": 931, + "\"]}}<|": 932, + "ount": 933, + "Ġrecord": 934, + "umb": 935, + "Ġdik": 936, + "out": 937, + "Ġvalue": 938, + "़": 939, + "15": 940, + "Ġall": 941, + "Ġdikha": 942, + "ai": 943, + "Ġne": 944, + "Ġdikhao": 945, + "Ġrecords": 946, + "ahan": 947, + "Ġà¤ľà¤¹": 948, + "Ġyou": 949, + "Ġcolumn": 950, + "ध": 951, + "द": 952, + "ड": 953, + "Ġà¤ī": 954, + "Ġà¤ħ": 955, + "Ġjahan": 956, + "Ġ10": 957, + "Ġ>=": 958, + "à¤ļ": 959, + "Ġà¤Ķ": 960, + "Ġschema": 961, + "Ġda": 962, + "Ġdata": 963, + "ull": 964, + "ĠJ": 965, + "à¥ĭà¤Ĥ": 966, + "Ġent": 967, + "now": 968, + "Ġh": 969, + "Ġ3": 970, + "ries": 971, + "umbai": 972, + "UM": 973, + "ĠSUM": 974, + "Ġentries": 975, + "elhi": 976, + "');": 977, + "');\"}}<|": 978, + "OW": 979, + "ĠLOW": 980, + "ĠLOWER": 981, + "ND": 982, + "ĠAND": 983, + "LL": 984, + "ULL": 985, + "ĠIS": 986, + "ĠNULL": 987, + "Ġneed": 988, + "Ġ500": 989, + "Ġ<": 990, + "Ġब": 991, + "ĠDES": 992, + "ĠDESC": 993, + "व": 994, + "SC": 995, + "ĠASC": 996, + "OIN": 997, + "ĠON": 998, + "ĠJOIN": 999, + "IT": 1000, + "IMIT": 1001, + "ĠLIMIT": 1002, + "tive": 1003, + "Ġà¤Ķर": 1004, + "lease": 1005, + "know": 1006, + "Ġun": 1007, + "lect": 1008, + "tent": 1009, + "Please": 1010, + "à¥ī": 1011, + "KT": 1012, + "]}": 1013, + "hist": 1014, + "tool": 1015, + "role": 1016, + "content": 1017, + "history": 1018, + "Ġli": 1019, + "Ġnull": 1020, + "ĠE": 1021, + "hat": 1022, + "tal": 1023, + "Ġaur": 1024, + "ith": 1025, + "20": 1026, + "Ġwith": 1027, + "Ġà¤Ĩ": 1028, + "outh": 1029, + "Ġsa": 1030, + "Ġtotal": 1031, + "pend": 1032, + "pending": 1033, + "Ġlow": 1034, + "Ġwo": 1035, + "orth": 1036, + "nic": 1037, + "ronic": 1038, + "ronics": 1039, + "une": 1040, + "lectronics": 1041, + "à¤ĩ": 1042, + "ya": 1043, + "Ġcount": 1044, + "cu": 1045, + "à¤Ĥ।": 1046, + "à¥Į": 1047, + "Ġthan": 1048, + "Reil": 1049, + "Reilly": 1050, + "Ġko": 1051, + "etch": 1052, + "known": 1053, + "Show": 1054, + "à¤ľ": 1055, + "पय": 1056, + "10": 1057, + "Ġna": 1058, + "Ġर": 1059, + "Ġhi": 1060, + "à¥ĥ": 1061, + "Ġup": 1062, + "ur": 1063, + "à¥Īà¤Ĥ": 1064, + "Ġdo": 1065, + "Ġhig": 1066, + "à¤Ń": 1067, + "Ġlist": 1068, + "Ġà¤ļ": 1069, + "Ġor": 1070, + "Ġpl": 1071, + "Ġसà¤Ń": 1072, + "Ġड": 1073, + "Ġय": 1074, + "Ġshow": 1075, + "ich": 1076, + "qual": 1077, + "Ġequal": 1078, + "श": 1079, + "ease": 1080, + "Ġat": 1081, + "ter": 1082, + "Ġav": 1083, + "Ġla": 1084, + "Ġhave": 1085, + "Ġjoin": 1086, + "Ġ50": 1087, + "Ġhigh": 1088, + "Ġkar": 1089, + "Ġret": 1090, + "Ġlowest": 1091, + "Delhi": 1092, + "What": 1093, + "));\",\"": 1094, + "Mumbai": 1095, + "Ġthat": 1096, + "Ġcon": 1097, + "Ġu": 1098, + "Ġtool": 1099, + "urn": 1100, + "Ġreturn": 1101, + "11": 1102, + "202": 1103, + "ĠH": 1104, + "Ġsee": 1105, + "hich": 1106, + "display": 1107, + "ada": 1108, + "ld": 1109, + "ould": 1110, + "तम": 1111, + "बस": 1112, + "रम": 1113, + "ĠMIN": 1114, + "à¥Ģà¤Ĥ": 1115, + "नà¤ķ": 1116, + "ĠNO": 1117, + "ĠNOT": 1118, + "ष": 1119, + "',": 1120, + "VG": 1121, + "ĠAVG": 1122, + "ail": 1123, + "Ġ<=": 1124, + "Ġhain": 1125, + "Ġ('": 1126, + "Ġ1000": 1127, + "%');\"}}<|": 1128, + "'%": 1129, + "('%": 1130, + "EE": 1131, + "ET": 1132, + "IKE": 1133, + "WEE": 1134, + "ĠBET": 1135, + "ĠLIKE": 1136, + "WEEN": 1137, + "ĠBETWEEN": 1138, + "ge": 1139, + "Ġ10000": 1140, + "EAR": 1141, + "YEAR": 1142, + "AV": 1143, + "ING": 1144, + "ĠHAV": 1145, + "ĠHAVING": 1146, + "Ġ200": 1147, + "IS": 1148, + "TIN": 1149, + "ĠDIS": 1150, + "TINCT": 1151, + "ĠDISTINCT": 1152, + "Ġpre": 1153, + "vi": 1154, + "Ġnahi": 1155, + "Un": 1156, + "Ġmat": 1157, + "ik": 1158, + "Ġ15": 1159, + "AX": 1160, + "ĠMAX": 1161, + "Ġwal": 1162, + "Ġhighest": 1163, + "Ġनह": 1164, + "ous": 1165, + "Ġprevi": 1166, + "Ġprevious": 1167, + "Ġnik": 1168, + "Ġà¤Ł": 1169, + "count": 1170, + "Ġle": 1171, + "ror": 1172, + "ACT": 1173, + "RACT": 1174, + "XTRACT": 1175, + "ĠEXTRACT": 1176, + "ONT": 1177, + "ONTH": 1178, + "Ġत": 1179, + "Ġwa": 1180, + "customer": 1181, + "wer": 1182, + "nswer": 1183, + "Er": 1184, + "à¥Ī?": 1185, + "ĠInspect": 1186, + "ण": 1187, + "Unknown": 1188, + "ĠMumbai": 1189, + "à¤Ī": 1190, + "Ġbe": 1191, + "Can": 1192, + "\":\"/": 1193, + "Ġplease": 1194, + "ust": 1195, + "Ġas": 1196, + "answer": 1197, + "}}]}": 1198, + "Ġg": 1199, + "ĠDelhi": 1200, + "TKT": 1201, + "ĠTKT": 1202, + "The": 1203, + "Ġ5000": 1204, + "Ġz": 1205, + "rue": 1206, + "true": 1207, + "isEr": 1208, + ".\"}]}": 1209, + "isError": 1210, + "Ġcha": 1211, + "ach": 1212, + "Ġeach": 1213, + "Ġnikal": 1214, + "active": 1215, + "Ġ0": 1216, + "Ġà¤ħध": 1217, + "Ġ1500": 1218, + "à¤Ĺ": 1219, + "Ġmo": 1220, + "ut": 1221, + "South": 1222, + "ues": 1223, + "Ġvalues": 1224, + "North": 1225, + "pull": 1226, + "Pune": 1227, + "à¤ģ।": 1228, + "Books": 1229, + "Electronics": 1230, + "hiye": 1231, + "Ġchahiye": 1232, + "Ġà¤ķम": 1233, + "cend": 1234, + "Ġya": 1235, + "cending": 1236, + "ĠSouth": 1237, + "Ġreturned": 1238, + "can": 1239, + "fi": 1240, + "ढ": 1241, + "Ġactive": 1242, + "ĠSQL": 1243, + "yada": 1244, + "Ġzyada": 1245, + "Ġpending": 1246, + "Ġconn": 1247, + "ĠElectronics": 1248, + "ĠNorth": 1249, + "ाà¤Ĥ": 1250, + "ĠPune": 1251, + "res": 1252, + "Ġà¤Ĺ": 1253, + "Ġconnec": 1254, + "Ġपर": 1255, + "Ġtop": 1256, + "im": 1257, + "''": 1258, + "ny": 1259, + "show": 1260, + "Ġà¤Ĩप": 1261, + "amount": 1262, + "Ġkya": 1263, + "à¥Ī,": 1264, + "Ġgroups": 1265, + "Ġसबस": 1266, + "Ġlao": 1267, + "12": 1268, + "ĠBooks": 1269, + "th": 1270, + "Ġi": 1271, + "imum": 1272, + "Ġbest": 1273, + "Fetch": 1274, + "cost": 1275, + "Ġ/": 1276, + "Display": 1277, + "Ġincl": 1278, + "ilter": 1279, + "à¤Ŀ": 1280, + "ive": 1281, + "Ġspec": 1282, + "Ġspeci": 1283, + "Ġout": 1284, + "Ġconnection": 1285, + "ne": 1286, + "़ा": 1287, + "ine": 1288, + "uj": 1289, + "ujhe": 1290, + "Ġliye": 1291, + "Ġmin": 1292, + "Ġitems": 1293, + "balance": 1294, + ".\"}<|": 1295, + "लम": 1296, + "Ġnikalo": 1297, + "Ġà¤īन": 1298, + "Ġwant": 1299, + "Ġlike": 1300, + "Ġex": 1301, + "age": 1302, + "score": 1303, + "Ġpe": 1304, + "fore": 1305, + "à¤ĩà¤ı": 1306, + "ted": 1307, + "onth": 1308, + "give": 1309, + "Ġmonth": 1310, + "very": 1311, + "Ġevery": 1312, + "kh": 1313, + "Ġmatch": 1314, + "Ġare": 1315, + "Ġcase": 1316, + "able": 1317, + "Ġmany": 1318, + "Ġsh": 1319, + "लब": 1320, + "Ġkam": 1321, + "Ġjin": 1322, + "Ġsor": 1323, + "price": 1324, + "रद": 1325, + "बद": 1326, + "salary": 1327, + "set": 1328, + "Ġ2024": 1329, + "Ġkit": 1330, + "Ġwali": 1331, + "Ġmean": 1332, + "Ġ2026": 1333, + "erage": 1334, + "Ġaverage": 1335, + "are": 1336, + "Ġavail": 1337, + "Ġsorted": 1338, + "Ġba": 1339, + "Ġbefore": 1340, + "Ġà¤ĸ": 1341, + "13": 1342, + "Ġmore": 1343, + "Ġjust": 1344, + "Ġavailable": 1345, + "umber": 1346, + "Ġwhich": 1347, + "ck": 1348, + "ent": 1349, + "Ġpull": 1350, + "à¥Īà¤Ĥ?": 1351, + "ry": 1352, + "Ġunknown": 1353, + "Ġफ": 1354, + "ĠYEAR": 1355, + "14": 1356, + "Ġhis": 1357, + "aab": 1358, + "Ġhisaab": 1359, + "Which": 1360, + "Mujhe": 1361, + "Ġnumber": 1362, + "Ġkarke": 1363, + "Ġsaare": 1364, + "ran": 1365, + "Ġ2022": 1366, + "18": 1367, + "तन": 1368, + "Ġश": 1369, + "irst": 1370, + "Ġfirst": 1371, + "Ġdoes": 1372, + "Find": 1373, + "नतम": 1374, + "Ġsaari": 1375, + "resent": 1376, + "Ġà¤īस": 1377, + "DM": 1378, + "md": 1379, + "READM": 1380, + "README": 1381, + "uns": 1382, + "Ġ2023": 1383, + "ct": 1384, + "Ġपह": 1385, + "ni": 1386, + "Ġget": 1387, + "à¤Ľ": 1388, + "19": 1389, + "por": 1390, + "16": 1391, + "MONTH": 1392, + "cs": 1393, + "sal": 1394, + "repor": 1395, + "csv": 1396, + "sales": 1397, + "reports": 1398, + "Out": 1399, + "Output": 1400, + "22": 1401, + "Ġst": 1402, + "Ġसम": 1403, + "17": 1404, + "Ġshould": 1405, + "ste": 1406, + "Ġthis": 1407, + "ĠK": 1408, + "Th": 1409, + "Ġinste": 1410, + "ĠWould": 1411, + "This": 1412, + "Ġinstead": 1413, + "Ġà¤Ĩपà¤ķ": 1414, + "Ġjink": 1415, + "Ġपढ": 1416, + "Ġsab": 1417, + "बल": 1418, + "21": 1419, + "Ġfilter": 1420, + "ĠMONTH": 1421, + "रदर": 1422, + "23": 1423, + "Ġmax": 1424, + "Ġminimum": 1425, + "Ġcounts": 1426, + "Ġ6": 1427, + "ast": 1428, + "Ġfetch": 1429, + "reater": 1430, + "Ġ2025": 1431, + "Ġgreater": 1432, + "Ġascending": 1433, + "Ġcelsius": 1434, + "Ġless": 1435, + "Ġfahrenheit": 1436, + "Ġfail": 1437, + "Ġपहल": 1438, + "Ġà¤ħन": 1439, + "रण": 1440, + "the": 1441, + "Ġpeh": 1442, + "Ġpehle": 1443, + "Ġà¤ı": 1444, + "pur": 1445, + "son": 1446, + "ucknow": 1447, + "Ġ7": 1448, + "Ġit": 1449, + "Give": 1450, + "not": 1451, + "txt": 1452, + "setup": 1453, + "notes": 1454, + "à¤ĵ": 1455, + "fied": 1456, + "Ġspecified": 1457, + "णन": 1458, + "Ġà¤Ĺणन": 1459, + "ki": 1460, + "aipur": 1461, + "Ġबड": 1462, + "Ġequals": 1463, + "Ġà¤īसस": 1464, + "mall": 1465, + "Ġsmall": 1466, + "Ġjoins": 1467, + "Ġबत": 1468, + "sta": 1469, + "json": 1470, + "confi": 1471, + "config": 1472, + "Ġmatches": 1473, + "Ġdes": 1474, + "Ġà¤Ĩध": 1475, + "Ġho": 1476, + "ation": 1477, + "थ": 1478, + "Ġkitni": 1479, + "check": 1480, + "Ġwale": 1481, + "vel": 1482, + "rity": 1483, + "Ġsecu": 1484, + "Ġà¤īप": 1485, + "Ġlevel": 1486, + "Ġsecurity": 1487, + "Ġà¤īपलब": 1488, + "ff": 1489, + "Ġjinka": 1490, + "Ġonly": 1491, + "Ġbut": 1492, + "Ġja": 1493, + "Ġye": 1494, + "Ġpresent": 1495, + "Ġleast": 1496, + "Ġdescending": 1497, + "à¤ĸन": 1498, + "ho": 1499, + "Ġback": 1500, + "Ġmaine": 1501, + "Ġsabse": 1502, + "Ġwri": 1503, + "PI": 1504, + "ntic": 1505, + "what": 1506, + "Ġauthe": 1507, + "ntication": 1508, + "Ġauthentication": 1509, + "tail": 1510, + "Ġdetail": 1511, + "Ġdetails": 1512, + "Ġtry": 1513, + "नम": 1514, + "à¥ĩà¤Ĥ,": 1515, + "Ġcheck": 1516, + "Ġsort": 1517, + "please": 1518, + "ups": 1519, + "Ġbackups": 1520, + "Ġno": 1521, + "Ġdis": 1522, + "वल": 1523, + "Ġsmallest": 1524, + "Ġab": 1525, + "xes": 1526, + "dexes": 1527, + "om": 1528, + "ap": 1529, + "Ġag": 1530, + "Ġpls": 1531, + "à¤ķर": 1532, + "late": 1533, + "Ġbot": 1534, + "Ġhas": 1535, + "à¤ħ": 1536, + "à¤ŀ": 1537, + "Ġ9": 1538, + "Ġfi": 1539, + "Ġà¤īà¤ļ": 1540, + "शन": 1541, + "Ġà¤īनà¤ķ": 1542, + "Ġusing": 1543, + "Ġlast": 1544, + "à¤ļतम": 1545, + "tell": 1546, + "Lucknow": 1547, + "à¤ķत": 1548, + "one": 1549, + "Ġfind": 1550, + "Ġprovide": 1551, + "Ġpar": 1552, + "Ġgive": 1553, + "Ġ4": 1554, + "Ġwithin": 1555, + "tinct": 1556, + "Ġdistinct": 1557, + "Ġorde": 1558, + "Ġordered": 1559, + "fetch": 1560, + "Jaipur": 1561, + "Ġà¤Ĭ": 1562, + "start": 1563, + "eld": 1564, + "Ġhar": 1565, + "Ġfield": 1566, + "Ġkoi": 1567, + "Ġyear": 1568, + "am": 1569, + "ain": 1570, + "ure": 1571, + "Ġ12": 1572, + "Ġmaximum": 1573, + "Ġसà¤ķत": 1574, + "usive": 1575, + "Ġkauns": 1576, + "Ġoutput": 1577, + "Ġstu": 1578, + "Ġkaunsi": 1579, + "mes": 1580, + "Ġnames": 1581, + "Ġvi": 1582, + "ound": 1583, + "Ġbas": 1584, + "Ġhone": 1585, + "à¤ĩà¤Ł": 1586, + "06": 1587, + "Qu": 1588, + "created": 1589, + "ara": 1590, + "Query": 1591, + "ally": 1592, + "act": 1593, + "ata": 1594, + "Ġinclude": 1595, + "पर": 1596, + "range": 1597, + "Ġà¤Ĭपर": 1598, + "'t": 1599, + "ove": 1600, + "Ġabove": 1601, + "à¤ķà¤Ł": 1602, + "Ġà¤ĩ": 1603, + "Ġhow": 1604, + "Ġहर": 1605, + "Ġdocs": 1606, + "corded": 1607, + "recorded": 1608, + "Ġमह": 1609, + "Ġ2021": 1610, + "Ġlin": 1611, + "बर": 1612, + "Ġबर": 1613, + "Ġset": 1614, + "Ġà¤ıà¤ķ": 1615, + "lab": 1616, + "Ġmatlab": 1617, + "iqu": 1618, + "Ġuniqu": 1619, + "Ġunique": 1620, + "सत": 1621, + "Ġà¤Ķसत": 1622, + "Ġmost": 1623, + "Ġthem": 1624, + "ceed": 1625, + "Ġexceed": 1626, + "Ġus": 1627, + "Ġal": 1628, + "Ġfound": 1629, + "Ġinclusive": 1630, + "Ġstuff": 1631, + "Ġ8": 1632, + "Ġtell": 1633, + "Ġà¤ħà¤ļ": 1634, + "Ġfailed": 1635, + "Ġlarg": 1636, + "Pull": 1637, + "Ġà¤ĩस": 1638, + "Ġta": 1639, + "alcu": 1640, + "alculate": 1641, + "Ġjaha": 1642, + "een": 1643, + "ween": 1644, + "Ġbet": 1645, + "2026": 1646, + "Ġbetween": 1647, + "Ġusk": 1648, + "Ġar": 1649, + "़ि": 1650, + "Ġuske": 1651, + "Ġà¤ķरन": 1652, + "Ġbata": 1653, + "Ġbada": 1654, + "Ġpad": 1655, + "Ġà¤ķन": 1656, + "यह": 1657, + "सबस": 1658, + "Ġagain": 1659, + "तलब": 1660, + "Ġमतलब": 1661, + "Ġwhat": 1662, + "Ġkarne": 1663, + "Ġ11": 1664, + "Ġjo": 1665, + "02": 1666, + "à¤ĩल": 1667, + "2024": 1668, + "Ġspecify": 1669, + "Ġknow": 1670, + "Best": 1671, + "auns": 1672, + "ĠKauns": 1673, + "ĠKaunsa": 1674, + "Ġवह": 1675, + "08": 1676, + "Ġeit": 1677, + "Ġeither": 1678, + "ver": 1679, + "सम": 1680, + "Ġusse": 1681, + "2022": 1682, + "ao": 1683, + "à¤Łà¤°": 1684, + "à¤Ĥ।\"}<|": 1685, + "Tool": 1686, + "ew": 1687, + "eh": 1688, + "Yeh": 1689, + "usam": 1690, + "Ġmausam": 1691, + "Ġupar": 1692, + "ĠKya": 1693, + "arab": 1694, + "Ġbarab": 1695, + "Ġbarabar": 1696, + "ding": 1697, + "Ġfil": 1698, + "Ġlower": 1699, + "term": 1700, + "Ġsql": 1701, + "khne": 1702, + "50": 1703, + "Ġlikhne": 1704, + "Ġcan": 1705, + "ause": 1706, + "रव": 1707, + "Ġover": 1708, + "Ġfilte": 1709, + "Ġlook": 1710, + "Ġselect": 1711, + "Ġबढ": 1712, + "nsi": 1713, + "रमबद": 1714, + "फ": 1715, + "Ġà¤ħà¤ľ": 1716, + "ose": 1717, + "Ġstar": 1718, + "2023": 1719, + "01": 1720, + "How": 1721, + "2025": 1722, + "Ġan": 1723, + "Ġsum": 1724, + "़à¥Ģ": 1725, + "Ġif": 1726, + "Ġstarting": 1727, + "à¤ī": 1728, + "Ġsearch": 1729, + "Ġjoining": 1730, + "Ġins": 1731, + "Ġdekh": 1732, + "Ġwriting": 1733, + "Ġunki": 1734, + "termine": 1735, + "Ġ13": 1736, + "obara": 1737, + "tom": 1738, + "Ġones": 1739, + "Ġbottom": 1740, + "uding": 1741, + "Ġwe": 1742, + "|>/": 1743, + "ा।": 1744, + "04": 1745, + "just": 1746, + "Ġupper": 1747, + "ete": 1748, + "Ġà¤Ĩà¤ĩà¤Ł": 1749, + "Ġà¤Ĩà¤ĩà¤Łà¤®": 1750, + "ving": 1751, + "Ġfiltered": 1752, + "Ġra": 1753, + "uch": 1754, + "Filter": 1755, + "à¤ĩस": 1756, + "Ġch": 1757, + "Ġthose": 1758, + "'s": 1759, + "ked": 1760, + "Ġbase": 1761, + "Ġdon": 1762, + "Ġà¤Ľ": 1763, + "हत": 1764, + "API": 1765, + "ĠAPI": 1766, + "Ġunhe": 1767, + "à¤ĩप": 1768, + "gn": 1769, + "रत": 1770, + "Ġid": 1771, + "Ġign": 1772, + "Ġres": 1773, + "Ġdataset": 1774, + "ically": 1775, + "Ġso": 1776, + "Ġbad": 1777, + "ag": 1778, + "Ġlargest": 1779, + "Ġdh": 1780, + "ecause": 1781, + "Ġbecause": 1782, + "Ġtak": 1783, + "jah": 1784, + "la": 1785, + "Ġwajah": 1786, + "Ġbaad": 1787, + "elect": 1788, + "Ġincluding": 1789, + "05": 1790, + "indexes": 1791, + "Ġindexes": 1792, + "Select": 1793, + "nder": 1794, + "Ġhigher": 1795, + "Ġunder": 1796, + "à¤ķतम": 1797, + "लà¤Ĺ": 1798, + "Bring": 1799, + "Ġतà¤ķ": 1800, + "Ġlim": 1801, + "03": 1802, + "Ġwrite": 1803, + "Ġbased": 1804, + "find": 1805, + "à¤īन": 1806, + "09": 1807, + "rict": 1808, + "Ġstrict": 1809, + "Ġstrictly": 1810, + "need": 1811, + "Ġthere": 1812, + "Ġper": 1813, + "Kri": 1814, + "pya": 1815, + "Kripya": 1816, + "Present": 1817, + "रस": 1818, + "रय": 1819, + "fter": 1820, + "Ġfull": 1821, + "ĠDobara": 1822, + "Ġlines": 1823, + "tify": 1824, + "De": 1825, + "07": 1826, + "Ġaap": 1827, + "Ġà¤ħà¤ķ": 1828, + "ाà¤Ĥ,": 1829, + "Ġco": 1830, + "Determine": 1831, + "unki": 1832, + "mplete": 1833, + "षर": 1834, + "Ġcomplete": 1835, + "yunki": 1836, + "Ġdate": 1837, + "tain": 1838, + "ated": 1839, + "Ġek": 1840, + "à¥Ĥà¤Ĥ": 1841, + "Ġcontain": 1842, + "़à¥ĩà¤Ĥ।": 1843, + "Ġsaath": 1844, + "heck": 1845, + "à¤Ľà¤²": 1846, + "Ġview": 1847, + "Ġथ": 1848, + "Ġok": 1849, + "uru": 1850, + "Ġshuru": 1851, + "ong": 1852, + "Ġlo": 1853, + "Ġhaving": 1854, + "Ġexact": 1855, + "Ġrange": 1856, + "Ġany": 1857, + "Ġयह": 1858, + "Ġkitna": 1859, + "Ġiden": 1860, + "kin": 1861, + "Ġlekin": 1862, + "atao": 1863, + "Kya": 1864, + "iye": 1865, + "Ġafter": 1866, + "Ġexist": 1867, + "Ġ14": 1868, + "Ġnow": 1869, + "Ġsame": 1870, + "Ġbasically": 1871, + "Ġbatao": 1872, + "Ġju": 1873, + "Is": 1874, + "Ġcap": 1875, + "Ext": 1876, + "ract": 1877, + "Ġsure": 1878, + "Extract": 1879, + "ig": 1880, + "Ġbig": 1881, + "Ġvia": 1882, + "Ġarrange": 1883, + "Da": 1884, + "kte": 1885, + "rin": 1886, + "Ġsakte": 1887, + "Ġbeing": 1888, + "Database": 1889, + "rint": 1890, + "Ġdiff": 1891, + "case": 1892, + "Ġboth": 1893, + "à¥ĭ।": 1894, + "Ġgiven": 1895, + "वरण": 1896, + "Ġmatching": 1897, + "Ġdid": 1898, + "मल": 1899, + "department": 1900, + "ranked": 1901, + "Ġwere": 1902, + "und": 1903, + "Ġpadho": 1904, + "nsitive": 1905, + "Ġdhund": 1906, + "ince": 1907, + "़à¥ĩ": 1908, + "़à¥ĭà¤Ĥ": 1909, + "mp": 1910, + "तर": 1911, + "Ġà¤ķरà¤ķ": 1912, + "à¥Īà¤Ĥ,": 1913, + "फल": 1914, + "mpty": 1915, + "à¥Ģ।": 1916, + "Ġkarta": 1917, + "Ġbasis": 1918, + "ĠLucknow": 1919, + "Could": 1920, + "category": 1921, + "़à¥į": 1922, + "rang": 1923, + "Ġarrang": 1924, + "ek": 1925, + "ech": 1926, + "ĠJaipur": 1927, + "Ġbeech": 1928, + "Pich": 1929, + "Ġfal": 1930, + "Ġfall": 1931, + "Calculate": 1932, + "Ġdiya": 1933, + "Ġcalculate": 1934, + "Ġfalls": 1935, + "tyek": 1936, + "Ġpra": 1937, + "oints": 1938, + "ĠHow": 1939, + "Ġpratyek": 1940, + "hey": 1941, + "Ġवर": 1942, + "Ġhu": 1943, + "नत": 1944, + "Ġer": 1945, + "Ġtoh": 1946, + "Ġidentify": 1947, + "iz": 1948, + "दस": 1949, + "Ġदस": 1950, + "Ġबन": 1951, + "Ġà¤Ĺà¤Ī": 1952, + "inti": 1953, + "Ġginti": 1954, + "Ġalong": 1955, + "Ġgay": 1956, + "Ġloc": 1957, + "Ġmuch": 1958, + "Ġpa": 1959, + "Ġjoined": 1960, + "ds": 1961, + "gh": 1962, + "Ġretrieve": 1963, + "100": 1964, + "Ġrun": 1965, + "ating": 1966, + "Ġढ": 1967, + "Ġà¤¹à¤Ł": 1968, + "Ġdiffe": 1969, + "Ġsabhi": 1970, + "Print": 1971, + "ped": 1972, + "uplic": 1973, + "Ġbel": 1974, + "Ġbadh": 1975, + "fo": 1976, + "Ġinfo": 1977, + "Ġbadhte": 1978, + "Ġend": 1979, + "Ġwas": 1980, + "Ġexceeds": 1981, + "Ġalag": 1982, + "Ġchho": 1983, + "Ġhue": 1984, + "Ġhaven": 1985, + "Ġranked": 1986, + "à¤ķरण": 1987, + "à¤ĥ": 1988, + "Ġdikh": 1989, + "Ġmake": 1990, + "Ġlaao": 1991, + "Ġrakh": 1992, + "Ġexceeding": 1993, + "Ġà¤Ń": 1994, + "Ġbana": 1995, + "Ġrakhte": 1996, + "Ġà¤Ĺय": 1997, + "Ġignore": 1998, + "Ġdue": 1999, + "Ġjaan": 2000, + "aal": 2001, + "Ġnikaal": 2002, + "sensitive": 2003, + "Ġlooking": 2004, + "Ġbanao": 2005, + "Ġarranged": 2006, + "Ġसर": 2007, + "Look": 2008, + "ks": 2009, + "support": 2010, + "Ġwoh": 2011, + "Pichla": 2012, + "66": 2013, + "ĠPlease": 2014, + "pective": 2015, + "pt": 2016, + "Ġincluded": 2017, + "Ġलà¤Ĺ": 2018, + "Ġam": 2019, + "सप": 2020, + "Ġरà¤ĸ": 2021, + "Ġ60": 2022, + "status": 2023, + "Ġpu": 2024, + "ide": 2025, + "Ġbhi": 2026, + "Ġinside": 2027, + "Ġcle": 2028, + "Ġpoints": 2029, + "Ġdele": 2030, + "Ġ103": 2031, + "Ġà¤īसà¤ķ": 2032, + "Ġdekho": 2033, + "Ġpura": 2034, + "mujhe": 2035, + "tempt": 2036, + "irf": 2037, + "Ġattempt": 2038, + "à¥Īà¤Ĥ।": 2039, + "Check": 2040, + "Last": 2041, + "Ġ16": 2042, + "Ġjuda": 2043, + "retrieve": 2044, + "king": 2045, + "Ġsto": 2046, + "reating": 2047, + "Ġsan": 2048, + "khya": 2049, + "Ġstored": 2050, + "Ġsankhya": 2051, + "Ġerror": 2052, + "hing": 2053, + "itni": 2054, + "Ġfields": 2055, + "Ġkyunki": 2056, + "Ġwithout": 2057, + "Ġlimits": 2058, + "thing": 2059, + "Ma": 2060, + "Ġà¤ļल": 2061, + "Ġtrying": 2062, + "Ġbigg": 2063, + "Maine": 2064, + "Ġtha": 2065, + "Ġà¤ħलà¤Ĺ": 2066, + "Just": 2067, + "region": 2068, + "à¤ħलà¤Ĺ": 2069, + "Ġ600": 2070, + "57": 2071, + "gar": 2072, + "quen": 2073, + "Ġsequen": 2074, + "Ġexactly": 2075, + "Ġsequence": 2076, + "iated": 2077, + "soc": 2078, + "vity": 2079, + "tivity": 2080, + "Ġpo": 2081, + "Ġsensi": 2082, + "Ġassoc": 2083, + "Ġsensitivity": 2084, + "Ġassociated": 2085, + "oring": 2086, + "रà¤ķ": 2087, + "Ġà¤Ĩà¤ķ": 2088, + "à¤ľà¤¹": 2089, + "Ġgo": 2090, + "Ġeverything": 2091, + "Ġdont": 2092, + "Ġsaara": 2093, + "Ġlarger": 2094, + "Ġignoring": 2095, + "Ġlimit": 2096, + "फलत": 2097, + "Ġinspect": 2098, + "lind": 2099, + "à¥Ĥà¤ģ।": 2100, + "वह": 2101, + "Ġens": 2102, + "Ġgaya": 2103, + "Ġends": 2104, + "à¤Ĥ,": 2105, + "à¤ģ,": 2106, + "Ġdidn": 2107, + "yan": 2108, + "Ġot": 2109, + "Ġcom": 2110, + "Ġrespective": 2111, + "Ġother": 2112, + "Ġwh": 2113, + "Ġ55": 2114, + "Wo": 2115, + "tents": 2116, + "Ġà¤ķरत": 2117, + "Ġcontents": 2118, + "Ġjinke": 2119, + "Ġel": 2120, + "à¤ł": 2121, + "alc": 2122, + "ole": 2123, + "Ġcalc": 2124, + "Ġwhole": 2125, + "Ġelse": 2126, + "Count": 2127, + "Ġuse": 2128, + "Ġwahan": 2129, + "Ġlate": 2130, + "Ġokay": 2131, + "Ġlater": 2132, + "tart": 2133, + "asing": 2134, + "िà¤Ĥ": 2135, + "cre": 2136, + "Ġincre": 2137, + "Ġmatter": 2138, + "Ġoverall": 2139, + "Ġincreasing": 2140, + "Ġunke": 2141, + "Ġà¤Ĩर": 2142, + "alag": 2143, + "Provid": 2144, + "Ġbelow": 2145, + "Provide": 2146, + "Ġentry": 2147, + "lated": 2148, + "56": 2149, + "Documentation": 2150, + "Ġthen": 2151, + "93": 2152, + "96": 2153, + "gate": 2154, + "gregate": 2155, + "Ġaggregate": 2156, + "Ġhui": 2157, + "44": 2158, + "pas": 2159, + "Ġcontent": 2160, + "Ġwapas": 2161, + "92": 2162, + "Ġsuch": 2163, + "Ġempty": 2164, + "rand": 2165, + "Ġgrand": 2166, + "insensitive": 2167, + "Ġaay": 2168, + "Ġaage": 2169, + "पस": 2170, + "Ġexcl": 2171, + "Ġchhoti": 2172, + "Ġdb": 2173, + "Ġpata": 2174, + "Ġpadh": 2175, + "Batao": 2176, + "Ġremo": 2177, + "Ġदर": 2178, + "Ġà¤Ĩà¤Ĺ": 2179, + "ĠFind": 2180, + "Ġaayin": 2181, + "89": 2182, + "à¤Ĥ?": 2183, + "ipe": 2184, + "Ġà¤ħव": 2185, + "Ġsaal": 2186, + "Ġdhundo": 2187, + "Be": 2188, + "aro": 2189, + "Ġsince": 2190, + "Ġzaro": 2191, + "Ġjinki": 2192, + "Before": 2193, + "Ġnon": 2194, + "Ġपत": 2195, + "Ġ1600": 2196, + "Ġरह": 2197, + "Ġcontains": 2198, + "Docs": 2199, + "Ġ[": 2200, + "Ġmujhe": 2201, + "à¥ĩ।": 2202, + "Ġthanks": 2203, + "Ġkitne": 2204, + "Ġdelete": 2205, + "nda": 2206, + "pute": 2207, + "ounda": 2208, + "uno": 2209, + "Ġkarna": 2210, + "Ġchuno": 2211, + "Ġरà¤ĸत": 2212, + "Ġsha": 2213, + "Ġ110": 2214, + "ompute": 2215, + "Ġlowercase": 2216, + "87": 2217, + "Kyunki": 2218, + "ital": 2219, + "Ġध": 2220, + "Ġbounda": 2221, + "Ġcapital": 2222, + "Ġthi": 2223, + "ĠJust": 2224, + "Ġपहà¤ļ": 2225, + "Ġuppercase": 2226, + "27": 2227, + "pen": 2228, + "ĠWo": 2229, + "Ġ53": 2230, + "Ġ505": 2231, + "Ġcapitaliz": 2232, + "Ġaccord": 2233, + "Ġavg": 2234, + "Ġlocate": 2235, + "Ġaccording": 2236, + "Ġinter": 2237, + "Ġkaun": 2238, + "Ġspecif": 2239, + "500": 2240, + "Table": 2241, + "Ġ5010": 2242, + "Ġfailure": 2243, + "Ġraha": 2244, + "49": 2245, + "Kitni": 2246, + "à¥ĭ,": 2247, + "ces": 2248, + "filter": 2249, + "kind": 2250, + "Ġmaking": 2251, + "Ġgoing": 2252, + "Ġboundary": 2253, + "47": 2254, + "Ġtreating": 2255, + "ĠCheck": 2256, + "Ġduplic": 2257, + "Ġ52": 2258, + "Ġ150": 2259, + "33": 2260, + "Since": 2261, + "Vi": 2262, + "रह": 2263, + "Ġbiggest": 2264, + "View": 2265, + "ul": 2266, + "roke": 2267, + "Ġbroke": 2268, + "Ġlinked": 2269, + "Ġcontained": 2270, + "Ġdifferent": 2271, + "ula": 2272, + "Ġcum": 2273, + "ulative": 2274, + "Ġcumulative": 2275, + "Ġwor": 2276, + "nce": 2277, + "formance": 2278, + "Ġ2000": 2279, + "Ġperformance": 2280, + "Ġ10010": 2281, + "ढन": 2282, + "ntion": 2283, + "à¤Łà¤²": 2284, + "Ġconnected": 2285, + "most": 2286, + "aye": 2287, + "trict": 2288, + "Erase": 2289, + "Ġtara": 2290, + "Ġrestrict": 2291, + "Empty": 2292, + "hon": 2293, + "uped": 2294, + "Ġgro": 2295, + "nthon": 2296, + "Ġpanthon": 2297, + "Ġgrouped": 2298, + "Ġpart": 2299, + "Ġnothing": 2300, + "98": 2301, + "Ġen": 2302, + "pply": 2303, + "Ġjis": 2304, + "Ġà¤īत": 2305, + "Ġ102": 2306, + "Ġlag": 2307, + "Ġdono": 2308, + "53": 2309, + "Jahan": 2310, + "Tally": 2311, + "pass": 2312, + "Ġsur": 2313, + "Ġlet": 2314, + "Ġ550": 2315, + "passes": 2316, + "Ġsurpasses": 2317, + "ates": 2318, + "Ġpri": 2319, + "Ġà¤ĩसल": 2320, + "37": 2321, + "Main": 2322, + "Ġ70": 2323, + "Ġhoon": 2324, + "Ġcompletely": 2325, + "Ġcapped": 2326, + "Ġensure": 2327, + "42": 2328, + "pon": 2329, + "ell": 2330, + "Ġblind": 2331, + "Ġसह": 2332, + "Ġsaid": 2333, + "Ġसमà¤Ĺ": 2334, + "Ġ700": 2335, + "Ġabout": 2336, + "Ġdhyan": 2337, + "ote": 2338, + "olds": 2339, + "Ġcor": 2340, + "ĠNote": 2341, + "Ġà¤īपय": 2342, + "Ġholds": 2343, + "respon": 2344, + "Ġyet": 2345, + "Ġcorrespon": 2346, + "Ġindi": 2347, + "ashi": 2348, + "Ġfashi": 2349, + "Ġplz": 2350, + "Ġfashion": 2351, + "72": 2352, + "Go": 2353, + "Tell": 2354, + "hai": 2355, + "Ġà¤ħà¤Ĺ": 2356, + "Ġdifferen": 2357, + "Ġnikaalo": 2358, + "Ġà¤ħवर": 2359, + "bin": 2360, + "Ġcho": 2361, + "Ġkey": 2362, + "Ġoptions": 2363, + "output": 2364, + "Ġlocated": 2365, + "Ġcombin": 2366, + "Ġcombined": 2367, + "No": 2368, + "ify": 2369, + "kari": 2370, + "tu": 2371, + "Ġti": 2372, + "Ġad": 2373, + "erify": 2374, + "Ġactu": 2375, + "Ġdetermine": 2376, + "Ġverify": 2377, + "Ġjaankari": 2378, + "Woh": 2379, + "Ġactually": 2380, + "li": 2381, + "ruc": 2382, + "ture": 2383, + "Ġseq": 2384, + "Ġ520": 2385, + "Ġkiya": 2386, + "à¤ļन": 2387, + "Ġkarte": 2388, + "Ġcondition": 2389, + "Ġstruc": 2390, + "Ġclear": 2391, + "Ġshare": 2392, + "Ġjisme": 2393, + "Ġadd": 2394, + "Ġstructure": 2395, + "Pur": 2396, + "Start": 2397, + "Ġgre": 2398, + "atest": 2399, + "Ġreading": 2400, + "Ġसप": 2401, + "Ġjya": 2402, + "Ġjudi": 2403, + "Purge": 2404, + "Ġgreatest": 2405, + "Ġjyaada": 2406, + "duplic": 2407, + "ori": 2408, + "to": 2409, + "ading": 2410, + "Ġca": 2411, + "à¤ķड": 2412, + "Ġleading": 2413, + "Ġjinme": 2414, + "Ġprior": 2415, + "Ġtime": 2416, + "Compute": 2417, + "Ġr": 2418, + "नन": 2419, + "Ġ300": 2420, + "Ġmode": 2421, + "Ġbaar": 2422, + "Ġremoving": 2423, + "aar": 2424, + "Ġrelated": 2425, + "usaar": 2426, + "Ġ101": 2427, + "Ġanusaar": 2428, + "Ġbade": 2429, + "52": 2430, + "58": 2431, + "68": 2432, + "ak": 2433, + "Ġcause": 2434, + "Ġcols": 2435, + "Ġdikhaiye": 2436, + "Ġ5050": 2437, + "षण": 2438, + "Ġpeak": 2439, + "Ġclean": 2440, + "63": 2441, + "Ġreport": 2442, + "hao": 2443, + "Ġretry": 2444, + "à¤Ĺर": 2445, + "Ġretrying": 2446, + "74": 2447, + "You": 2448, + "ka": 2449, + "look": 2450, + "Ġri": 2451, + "à¤Ĩ": 2452, + "Ġpich": 2453, + "़à¥ĩà¤Ĥ": 2454, + "Ġरà¤ĸन": 2455, + "ght": 2456, + "Ġright": 2457, + "62": 2458, + "यर": 2459, + "Ġà¤ĩसन": 2460, + "Agar": 2461, + "Ġkindly": 2462, + "ĠShow": 2463, + "Ġdobara": 2464, + "Ġà¤ĩसà¤ķ": 2465, + "Ġhua": 2466, + "Ide": 2467, + "To": 2468, + "ntify": 2469, + "Ġwipe": 2470, + "ane": 2471, + "Ġ31": 2472, + "वश": 2473, + "Ġà¤Ĩà¤Ī": 2474, + "Ġà¤Ĩवश": 2475, + "Ġzaroor": 2476, + "Identify": 2477, + "Del": 2478, + "Ġmil": 2479, + "Ġ1100": 2480, + "Delete": 2481, + "dl": 2482, + "val": 2483, + "hen": 2484, + "Ġregar": 2485, + "à¥Ģà¤Ĥ।": 2486, + "Ġ210": 2487, + "Ġinterval": 2488, + "Ġspecifically": 2489, + "dless": 2490, + "Ġregardless": 2491, + "eed": 2492, + "ss": 2493, + "ross": 2494, + "Ġacross": 2495, + "liye": 2496, + "ĠThen": 2497, + "Ġisliye": 2498, + "Ġ250": 2499, + "Ġnaam": 2500, + "à¤ħà¤Ĺर": 2501, + "Ġ510": 2502, + "Need": 2503, + "Ġ51": 2504, + "Ġ1050": 2505, + "bhai": 2506, + "ough": 2507, + "Ġhavent": 2508, + "Aa": 2509, + "Ġsirf": 2510, + "Ġunka": 2511, + "khri": 2512, + "Aakhri": 2513, + "ingle": 2514, + "ĠIsk": 2515, + "Ġsingle": 2516, + "Ġdhundho": 2517, + "war": 2518, + "ĠAnswer": 2519, + "Ġonwar": 2520, + "ĠBatao": 2521, + "Ġ5100": 2522, + "ĠIska": 2523, + "Ġonwards": 2524, + "batao": 2525, + "Ġ10500": 2526, + "ataiye": 2527, + "Ġcapitalization": 2528, + "Ġjismein": 2529, + "77": 2530, + "Ġ120": 2531, + "à¥ĭà¤Ĥ।": 2532, + "uis": 2533, + "tinguis": 2534, + "Ġà¤ħत": 2535, + "Ġdistinguis": 2536, + "Ġdistinguishing": 2537, + "IJस": 2538, + "Ġ501": 2539, + "Ġ92": 2540, + "Ġà¤IJस": 2541, + "मà¤ļ": 2542, + "Ġ10100": 2543, + "32": 2544, + "35": 2545, + "65": 2546, + "79": 2547, + "ĠCount": 2548, + "à¤īस": 2549, + "Ġjaata": 2550, + "Ġtarah": 2551, + "69": 2552, + "94": 2553, + "tire": 2554, + "ora": 2555, + "ice": 2556, + "Ġpao": 2557, + "à¤Ĺत": 2558, + "Ġpoora": 2559, + "Ġentire": 2560, + "59": 2561, + "Sum": 2562, + "Sirf": 2563, + "yein": 2564, + "Ġwahi": 2565, + "Ġdesc": 2566, + "Ġbatayein": 2567, + "Ġdekhna": 2568, + "Ġresul": 2569, + "Ġpaas": 2570, + "Ġbelong": 2571, + "kya": 2572, + "ĵर": 2573, + "sequ": 2574, + "Ġà¤ĵर": 2575, + "Ġsub": 2576, + "Ġkh": 2577, + "Ġ111": 2578, + "Ġmention": 2579, + "Ġà¤īल": 2580, + "Ġlies": 2581, + "Ġतरह": 2582, + "Ġletter": 2583, + "Ġdifferences": 2584, + "sequent": 2585, + "Ġsubsequent": 2586, + "31": 2587, + "88": 2588, + "Cle": 2589, + "hta": 2590, + "ijर": 2591, + "alanki": 2592, + "Ġkare": 2593, + "डर": 2594, + "Ġhalanki": 2595, + "Ġchahta": 2596, + "Ġjahaan": 2597, + "Ġduplicates": 2598, + "Ġwork": 2599, + "Ġresults": 2600, + "Clear": 2601, + "ali": 2602, + "tical": 2603, + "Ġlisted": 2604, + "Ġ45": 2605, + "Ġidentical": 2606, + "Ġ143": 2607, + "Jin": 2608, + "blind": 2609, + "pari": 2610, + "Ġcreating": 2611, + "dar": 2612, + "Ġpadhi": 2613, + "Ġandar": 2614, + "Ġbadi": 2615, + "़à¥ĩ।": 2616, + "Ġjaani": 2617, + "Ġcompari": 2618, + "Ġkhali": 2619, + "Ġcomparison": 2620, + "Ac": 2621, + "Bataiye": 2622, + "aki": 2623, + "select": 2624, + "Ġaao": 2625, + "pecting": 2626, + "Ġsensitive": 2627, + "सà¤ķ": 2628, + "ĠStart": 2629, + "Ġ502": 2630, + "Ġ1510": 2631, + "Ġnumbers": 2632, + "Ġjinko": 2633, + "Ġ62": 2634, + "Ġlinking": 2635, + "Ġinspecting": 2636, + "Ġchhota": 2637, + "Ġduplicate": 2638, + "43": 2639, + "Loc": 2640, + "Open": 2641, + "ories": 2642, + "ual": 2643, + "uring": 2644, + "head": 2645, + "Ġahead": 2646, + "Ġcateg": 2647, + "pulated": 2648, + "vidual": 2649, + "मà¤ķ": 2650, + "Ġbeen": 2651, + "Ġgot": 2652, + "Ġgayi": 2653, + "Ġpopulated": 2654, + "Ġensuring": 2655, + "Ġindividual": 2656, + "Ġkarein": 2657, + "Locate": 2658, + "Ġcategories": 2659, + "'d": 2660, + "lo": 2661, + "ugh": 2662, + "tiyan": 2663, + "rough": 2664, + "move": 2665, + "Remove": 2666, + "Ġpan": 2667, + "Ġflo": 2668, + "chtiyan": 2669, + "Ġपड": 2670, + "मत": 2671, + "Ġthrough": 2672, + "Ġdikhana": 2673, + "à¤ľà¤¨": 2674, + "Ġ503": 2675, + "à¤ħब": 2676, + "Ġ91": 2677, + "Ġtaaki": 2678, + "Ġagainst": 2679, + "Ġdikhaye": 2680, + "Ġbigger": 2681, + "Ġpanchtiyan": 2682, + "Ġfloor": 2683, + "29": 2684, + "gan": 2685, + "ning": 2686, + "od": 2687, + "Ġinsensitive": 2688, + "à¥ĩ,": 2689, + "Ġbring": 2690, + "Ġkho": 2691, + "itne": 2692, + "नश": 2693, + "यम": 2694, + "़à¥ĩà¤Ĥ,": 2695, + "दनश": 2696, + "à¤µà¤ľà¤¨": 2697, + "Ġorgan": 2698, + "Ġplus": 2699, + "Ġexam": 2700, + "Ġsorting": 2701, + "Ġdoesn": 2702, + "Ġalone": 2703, + "ized": 2704, + "Ġrunning": 2705, + "Ġcapitalized": 2706, + "Ġlagao": 2707, + "Ġdikhayein": 2708, + "Ġkhoj": 2709, + "Ġorganized": 2710, + "Ġexamine": 2711, + "40": 2712, + "75": 2713, + "Aap": 2714, + "Kitne": 2715, + "Sort": 2716, + "ana": 2717, + "ber": 2718, + "bsi": 2719, + "kul": 2720, + "thao": 2721, + "½Ð": 2722, + "нÐ": 2723, + "Ġapply": 2724, + "ा-": 2725, + "मнÐ": 2726, + "हर": 2727, + "ilkul": 2728, + "Ġuthao": 2729, + "Ġconnecting": 2730, + "ĠKitni": 2731, + "Ġyeh": 2732, + "à¤ħस": 2733, + "Ġlinks": 2734, + "Ġwebsi": 2735, + "Ġrahe": 2736, + "Ġcorresponding": 2737, + "Ġchote": 2738, + "मне": 2739, + "Ġwebsite": 2740, + "Apply": 2741, + "Wipe": 2742, + "ep": 2743, + "iod": 2744, + "mber": 2745, + "number": 2746, + "member": 2747, + "Ġremember": 2748, + "Ġwan": 2749, + "Ġmind": 2750, + "Ġbilkul": 2751, + "Ġprint": 2752, + "Ġkeep": 2753, + "Ġशहर": 2754, + "Ġperiod": 2755, + "Ġerase": 2756, + "Ġdeleting": 2757, + "duplicate": 2758, + "Ġwanna": 2759, + "Ġkeeping": 2760, + "54": 2761, + "Kind": 2762, + "cess": 2763, + "ppens": 2764, + "rab": 2765, + "Ġunit": 2766, + "Ġacche": 2767, + "यà¤ķत": 2768, + "Ġhata": 2769, + "Ġhappens": 2770, + "à¥Ĥà¤ģ": 2771, + "Ġà¤ħà¤Ń": 2772, + "Ġtotally": 2773, + "usively": 2774, + "Ġfiltering": 2775, + "Ġdated": 2776, + "Ġexclusively": 2777, + "Ġzaroori": 2778, + "Ġlagana": 2779, + "Ġcaring": 2780, + "Access": 2781, + "As": 2782, + "Vac": 2783, + "dy": 2784, + "ier": 2785, + "ior": 2786, + "mier": 2787, + "uper": 2788, + "Ġsuper": 2789, + "Ġinto": 2790, + "ा,": 2791, + "ani": 2792, + "रà¤ļन": 2793, + "Ġjane": 2794, + "Ġpremier": 2795, + "Ġchahe": 2796, + "Ġbaare": 2797, + "Ġstudy": 2798, + "Ġpaaye": 2799, + "Ġdikhate": 2800, + "Ġpadhni": 2801, + "Kindly": 2802, + "Vacate": 2803, + "Ġsuperior": 2804, + "points": 2805, + "sfy": 2806, + "Ġ+": 2807, + "tisfy": 2808, + "ayen": 2809, + "Ġlabels": 2810, + "Ġtoo": 2811, + "Ġneeds": 2812, + "Ġsatisfy": 2813, + "Ġà¤ļर": 2814, + "Unhi": 2815, + "imal": 2816, + "Ġminimal": 2817, + "Ġमहत": 2818, + "Ġ141": 2819, + "Ġendpoints": 2820, + "Ġdikhayen": 2821, + "Ġconditions": 2822, + "Ġbelongs": 2823, + "Ġdikhayenge": 2824, + "64": 2825, + "73": 2826, + "Join": 2827, + "ble": 2828, + "ga": 2829, + "sue": 2830, + "Ġpur": 2831, + "mantion": 2832, + "Ġà¤ijर": 2833, + "ayi": 2834, + "Ġissue": 2835, + "pls": 2836, + "Ġdocu": 2837, + "यव": 2838, + "दर": 2839, + "Ġdoing": 2840, + "Ġlaayi": 2841, + "Ġavailble": 2842, + "whats": 2843, + "à¤ħरà¤ķ": 2844, + "Ġfix": 2845, + "Ġremove": 2846, + "Ġcaused": 2847, + "Ġmilao": 2848, + "Ġpurge": 2849, + "Ġdocumantion": 2850, + "यवस": 2851, + "Stri": 2852, + "Ġterm": 2853, + "ntory": 2854, + "Ġshi": 2855, + "Ġsiz": 2856, + "Ġinve": 2857, + "Ġline": 2858, + "ĠSo": 2859, + "Ġ39": 2860, + "Ġthats": 2861, + "Ġतल": 2862, + "Ġ128": 2863, + "Ġexcluding": 2864, + "Ġpadhna": 2865, + "Ġcorresponds": 2866, + "Strip": 2867, + "Ġterms": 2868, + "Ġshirow": 2869, + "Ġsize": 2870, + "Ġinventory": 2871, + "We": 2872, + "].": 2873, + "go": 2874, + "nces": 2875, + "uk": 2876, + "Ġqui": 2877, + "Ġinsta": 2878, + "mployee": 2879, + "Ġkiye": 2880, + "Ġà¤ļढ": 2881, + "Ġasap": 2882, + "à¤Ľà¤¤": 2883, + "Pichli": 2884, + "Ġworry": 2885, + "Ġruk": 2886, + "Ġquick": 2887, + "Ġinstances": 2888, + "25": 2889, + "38": 2890, + "80": 2891, + "84": 2892, + "91": 2893, + "Run": 2894, + "hna": 2895, + "sirf": 2896, + "ured": 2897, + "Ġran": 2898, + "asured": 2899, + "Ġemployee": 2900, + "Ġonce": 2901, + "Ġdein": 2902, + "Ġmeasured": 2903, + "पम": 2904, + "à¥Ĥà¤Ĥ।": 2905, + "ikr": 2906, + "Ġतथ": 2907, + "Ġzikr": 2908, + "Ġ73": 2909, + "Ġnoting": 2910, + "Ġagle": 2911, + "Ġdhundhna": 2912, + "Ġarranging": 2913, + "Ġdikhani": 2914, + "Ġpichli": 2915, + "Ġrank": 2916, + "'.": 2917, + "61": 2918, + "ther": 2919, + "ĠRe": 2920, + "Ġwhether": 2921, + "voice": 2922, + "student": 2923, + "Ġmade": 2924, + "à¤ļय": 2925, + "Ġhit": 2926, + "view": 2927, + "Ġpadta": 2928, + "Ġjaanch": 2929, + "Ġexclude": 2930, + "Ġtaraf": 2931, + "Ġrid": 2932, + "ĠReview": 2933, + "39": 2934, + "ction": 2935, + "cates": 2936, + "ise": 2937, + "null": 2938, + "pres": 2939, + "sum": 2940, + "Ġstudent": 2941, + "ond": 2942, + "ina": 2943, + "ink": 2944, + "sed": 2945, + "section": 2946, + "ters": 2947, + "Ġbina": 2948, + "ĠLook": 2949, + "Ġthink": 2950, + "ĠGo": 2951, + "डà¤ķ": 2952, + "Ġà¤ħल": 2953, + "Ġgrab": 2954, + "Ġexpres": 2955, + "Ġ93": 2956, + "Ġcaps": 2957, + "Ġintersection": 2958, + "Ġspecific": 2959, + "Ġletters": 2960, + "Ġindicates": 2961, + "Ġpichla": 2962, + "Ġexpressed": 2963, + "ets": 2964, + "fin": 2965, + "ition": 2966, + "inest": 2967, + "Ġavo": 2968, + "Ġdefin": 2969, + "Ġfinest": 2970, + "Ġdho": 2971, + "Ġmeets": 2972, + "महत": 2973, + "Ġà¤µà¤ľà¤¹": 2974, + "à¥Ĥà¤ģ,": 2975, + "Ġkaran": 2976, + "Ġsmaller": 2977, + "Ġhota": 2978, + "Ġpoori": 2979, + "Ġà¤Ĩà¤ķल": 2980, + "Ġentity": 2981, + "Ġchoose": 2982, + "total": 2983, + "Ġcame": 2984, + "Top": 2985, + "itions": 2986, + "Ġavoid": 2987, + "Ġdefinitions": 2988, + "Ġdhoond": 2989, + "24": 2990, + "95": 2991, + "which": 2992, + "Ġdep": 2993, + "Ġपur": 2994, + "मà¤Ĺ": 2995, + "Ġkaise": 2996, + "Ġ59": 2997, + "Ġ104": 2998, + "Ġminus": 2999, + "Ġ63": 3000, + "Ġ66": 3001, + "Ġ44": 3002, + "Ġ89": 3003, + "Ġknown": 3004, + "duplicated": 3005, + "Ġzaroorat": 3006, + "Ġपurga": 3007, + "calculate": 3008, + "artments": 3009, + "Ġinvoice": 3010, + "रà¤ħ": 3011, + "यà¤ķ": 3012, + "Ġdisplaying": 3013, + "Ġthough": 3014, + "Ġà¤¨à¤ľ": 3015, + "Ġopen": 3016, + "111": 3017, + "Ġnikale": 3018, + "Ġconnect": 3019, + "Ġà¤ĸर": 3020, + "Ġabhi": 3021, + "Ġbataiye": 3022, + "Ġsamet": 3023, + "Ġdepartments": 3024, + "Ġà¤¨à¤ľà¤°à¤ħ": 3025, + "366": 3026, + "46": 3027, + "Ab": 3028, + "ake": 3029, + "ner": 3030, + "udes": 3031, + "ware": 3032, + "Ġstring": 3033, + "Ġaware": 3034, + "Ġbound": 3035, + "Ġ119": 3036, + "Ġ35": 3037, + "Ġincludes": 3038, + "085": 3039, + "Ġ139": 3040, + "Ġchal": 3041, + "Ġ148": 3042, + "Ġà¤ħà¤Ĺल": 3043, + "Ġquickly": 3044, + "Ġbounds": 3045, + "67": 3046, + "aao": 3047, + "bas": 3048, + "compute": 3049, + "tier": 3050, + "ubscription": 3051, + "itting": 3052, + "apply": 3053, + "à¤Łà¤®": 3054, + "à¤ħà¤ľ": 3055, + "Ġà¤ħवश": 3056, + "Ġchalaao": 3057, + "28": 3058, + "86": 3059, + "co": 3060, + "ied": 3061, + "mitting": 3062, + "nner": 3063, + "erating": 3064, + "Ġsubscription": 3065, + "plied": 3066, + "ubmitting": 3067, + "Ġसब": 3068, + "Ġ21": 3069, + "enerating": 3070, + "Ġsupplied": 3071, + "Ġmanner": 3072, + "Ġ107": 3073, + "Ġgenerating": 3074, + "though": 3075, + "Ġdisco": 3076, + "Ġ97": 3077, + "Ġ42": 3078, + "Ġalthough": 3079, + "Ġ132": 3080, + "Ġresubmitting": 3081, + "Ġ149": 3082, + "Ġdifference": 3083, + "Ġdiscover": 3084, + "97": 3085, + "after": 3086, + "ities": 3087, + "les": 3088, + "Ġtit": 3089, + "Ġcities": 3090, + "Ġtum": 3091, + "ayment": 3092, + "Ġsari": 3093, + "Ġproduc": 3094, + "Ġtitles": 3095, + "Ġproduct": 3096, + "48": 3097, + "Take": 3098, + "wo": 3099, + "Ġ122": 3100, + "Ġ30": 3101, + "Ġ37": 3102, + "124": 3103, + "Ġ40": 3104, + "Ġà¤ĩन": 3105, + "Ġ80": 3106, + "Ġ84": 3107, + "Ġ130": 3108, + "'?": 3109, + "36": 3110, + "ave": 3111, + "Ġir": 3112, + "यन": 3113, + "ययन": 3114, + "Ġgave": 3115, + "respective": 3116, + "हतर": 3117, + "Ġshared": 3118, + "Ġmilane": 3119, + "Ġirrespective": 3120, + "Ġ19": 3121, + "Ġ22": 3122, + "Ġ1010": 3123, + "Ġ98": 3124, + "Ġ5500": 3125, + "200": 3126, + "70": 3127, + "With": 3128, + "khao": 3129, + "Ġmust": 3130, + "à¤ĸर": 3131, + "Without": 3132, + "45": 3133, + "99": 3134, + "rse": 3135, + "Ġcou": 3136, + "Ġisn": 3137, + "hipment": 3138, + "à¤ķल": 3139, + "Ġkinds": 3140, + "Ġ17": 3141, + "Ġ113": 3142, + "Ġ58": 3143, + "Ġ105": 3144, + "Ġ106": 3145, + "Ġ38": 3146, + "143": 3147, + "148": 3148, + "Ġ47": 3149, + "Ġ48": 3150, + "Ġ136": 3151, + "pense": 3152, + "Ġcourse": 3153, + "It": 3154, + "1500": 3155, + "107": 3156, + "Ġexpense": 3157, + "120": 3158, + "323": 3159, + "603": 3160, + "800": 3161, + "Ġshipment": 3162, + "Ġ109": 3163, + "130": 3164, + "139": 3165, + "Ġ61": 3166, + "Ġ129": 3167, + "Ġ131": 3168, + "Ġà¤ĩनव": 3169, + "Ġpayment": 3170, + "Ġregions": 3171, + "Ġunhi": 3172, + "135": 3173, + "90": 3174, + "996": 3175, + "Ġ118": 3176, + "Ġ117": 3177, + "Ġ54": 3178, + "101": 3179, + "113": 3180, + "Ġ79": 3181, + "Ġ99": 3182, + "Ġ124": 3183, + "075": 3184, + "564": 3185, + "Ġlabeled": 3186, + "हà¤ķ": 3187, + "141": 3188, + "115": 3189, + "He": 3190, + "Ġaccount": 3191, + "Ġ36": 3192, + "106": 3193, + "128": 3194, + "Ġ138": 3195, + "Hey": 3196, + "Bhai": 3197, + "Ġwho": 3198, + "Ġthey": 3199, + "138": 3200, + "1000": 3201, + "10000": 3202, + "55": 3203, + "82": 3204, + "85": 3205, + "Courses": 3206, + "Ġratings": 3207, + "Ġ27": 3208, + "Ġà¤ħप": 3209, + "127": 3210, + "Ġ68": 3211, + "Ġ127": 3212, + "Ġ81": 3213, + "vice": 3214, + "Ġdevice": 3215, + "सà¤Ń": 3216, + "104": 3217, + "2021": 3218, + "Ġà¤Ĩà¤Īड": 3219, + "296": 3220, + "387": 3221, + "347": 3222, + "518": 3223, + "756": 3224, + "789": 3225, + "81": 3226, + "803": 3227, + "844": 3228, + "849": 3229, + "8366": 3230, + "966": 3231, + "ores": 3232, + "Ġ121": 3233, + "Ġ29": 3234, + "Ġ56": 3235, + "103": 3236, + "114": 3237, + "119": 3238, + "Ġ1550": 3239, + "1809": 3240, + "Ġ67": 3241, + "Ġ71": 3242, + "Ġ43": 3243, + "Ġ133": 3244, + "Ġ134": 3245, + "Ġ135": 3246, + "663": 3247, + "934": 3248, + "8757": 3249, + "2708": 3250, + "4652": 3251, + "3233": 3252, + "ike": 3253, + "tment": 3254, + "Ġvis": 3255, + "Ġappoin": 3256, + "dikhao": 3257, + "Ġbook": 3258, + "ĠLike": 3259, + "132": 3260, + "149": 3261, + "Ġvisit": 3262, + "Ġappointment": 3263, + "3564": 3264, + "414": 3265, + "453": 3266, + "600": 3267, + "606": 3268, + "722": 3269, + "944": 3270, + "972": 3271, + "aare": 3272, + "abhi": 3273, + "Ġsc": 3274, + "Ġ114": 3275, + "Ġdescriptions": 3276, + "Ġ25": 3277, + "Ġ28": 3278, + "Ġ57": 3279, + "Ġhon": 3280, + "Ġ32": 3281, + "Ġ33": 3282, + "षà¤ķ": 3283, + "122": 3284, + "142": 3285, + "Ġशब": 3286, + "Ġ65": 3287, + "Ġ69": 3288, + "Ġ74": 3289, + "Ġ75": 3290, + "Ġ76": 3291, + "Ġ77": 3292, + "Ġ90": 3293, + "Ġ96": 3294, + "Ġ49": 3295, + "Ġ126": 3296, + "Ġ87": 3297, + "Ġ137": 3298, + "076": 3299, + "Ġ142": 3300, + "Ġ144": 3301, + "Ġ145": 3302, + "Ġ147": 3303, + "924": 3304, + "899": 3305, + "4252": 3306, + "9402": 3307, + "9482": 3308, + "kha": 3309, + "led": 3310, + "Ġkuch": 3311, + "सब": 3312, + "Ġlikha": 3313, + "5000": 3314, + "Ġscores": 3315, + "41": 3316, + "136": 3317, + "Ġfilled": 3318, + "bataiye": 3319, + "तà¤ķ": 3320, + "लर": 3321, + "109": 3322, + "116": 3323, + "129": 3324, + "131": 3325, + "34": 3326, + "51": 3327, + "83": 3328, + "Or": 3329, + "Tic": 3330, + "der": 3331, + "à¤ijर": 3332, + "Ġdekhao": 3333, + "102": 3334, + "117": 3335, + "118": 3336, + "125": 3337, + "à¤ĩà¤Łà¤²": 3338, + "Order": 3339, + "Ticke": 3340, + "Employees": 3341, + "tabon": 3342, + "Ġà¤Ĺà¤ı": 3343, + "219": 3344, + "249": 3345, + "247": 3346, + "268": 3347, + "308": 3348, + "408": 3349, + "456": 3350, + "496": 3351, + "437": 3352, + "463": 3353, + "4138": 3354, + "502": 3355, + "533": 3356, + "5124": 3357, + "5107": 3358, + "5101": 3359, + "687": 3360, + "627": 3361, + "647": 3362, + "674": 3363, + "6085": 3364, + "6124": 3365, + "6130": 3366, + "709": 3367, + "758": 3368, + "762": 3369, + "788": 3370, + "7120": 3371, + "816": 3372, + "823": 3373, + "8113": 3374, + "8114": 3375, + "917": 3376, + "949": 3377, + "953": 3378, + "9085": 3379, + "9148": 3380, + "Sabhi": 3381, + "Ġ116": 3382, + "Ġ23": 3383, + "à¤Łà¤¸": 3384, + "Ġ10050": 3385, + "2063": 3386, + "à¤ĩन": 3387, + "1021": 3388, + "1072": 3389, + "1140": 3390, + "1227": 3391, + "1233": 3392, + "1321": 3393, + "1406": 3394, + "1857": 3395, + "1877": 3396, + "1824": 3397, + "1917": 3398, + "1931": 3399, + "1946": 3400, + "1625": 3401, + "2269": 3402, + "1792": 3403, + "1743": 3404, + "Ġ64": 3405, + "kitabon": 3406, + "Ġ88": 3407, + "1008": 3408, + "6622": 3409, + "6633": 3410, + "6641": 3411, + "578": 3412, + "5765": 3413, + "5600": 3414, + "5674": 3415, + "9318": 3416, + "9308": 3417, + "9374": 3418, + "9329": 3419, + "9620": 3420, + "4433": 3421, + "9250": 3422, + "9258": 3423, + "8704": 3424, + "2716": 3425, + "2789": 3426, + "2769": 3427, + "4900": 3428, + "4957": 3429, + "4789": 3430, + "4737": 3431, + "3310": 3432, + "5316": 3433, + "Ġprices": 3434, + "3715": 3435, + "3761": 3436, + "4211": 3437, + "4212": 3438, + "4262": 3439, + "7213": 3440, + "7216": 3441, + "7296": 3442, + "5293": 3443, + "5280": 3444, + "5861": 3445, + "6306": 3446, + "7457": 3447, + "7436": 3448, + "6263": 3449, + "3211": 3450, + "3265": 3451, + "3284": 3452, + "3517": 3453, + "7912": 3454, + "8835": 3455, + "4315": 3456, + "7598": 3457, + "5413": 3458, + "5433": 3459, + "3815": 3460, + "3865": 3461, + "3919": 3462, + "9520": 3463, + "3666": 3464, + "6798": 3465, + "8617": 3466, + "4828": 3467, + "3602": 3468, + "6035": 3469, + "8008": 3470, + "1159": 3471, + "2962": 3472, + "3871": 3473, + "3478": 3474, + "5183": 3475, + "7563": 3476, + "8035": 3477, + "8440": 3478, + "8497": 3479, + "9667": 3480, + "6630": 3481, + "2192": 3482, + "2490": 3483, + "2470": 3484, + "2683": 3485, + "3080": 3486, + "4374": 3487, + "5028": 3488, + "5333": 3489, + "6271": 3490, + "6473": 3491, + "6748": 3492, + "7093": 3493, + "7626": 3494, + "7880": 3495, + "8166": 3496, + "8236": 3497, + "9173": 3498, + "9491": 3499, + "9538": 3500, + "60": 3501, + "71": 3502, + "Jo": 3503, + "St": 3504, + "saare": 3505, + "Ġte": 3506, + "Ġword": 3507, + "udents": 3508, + "105": 3509, + "121": 3510, + "133": 3511, + "134": 3512, + "nsion": 3513, + "Ġanything": 3514, + "Ġjiska": 3515, + "Orders": 3516, + "Tickets": 3517, + "Students": 3518, + "Ġtension": 3519, + "216": 3520, + "266": 3521, + "244": 3522, + "269": 3523, + "2135": 3524, + "2924": 3525, + "322": 3526, + "306": 3527, + "350": 3528, + "304": 3529, + "305": 3530, + "357": 3531, + "368": 3532, + "3111": 3533, + "3085": 3534, + "3139": 3535, + "3075": 3536, + "3899": 3537, + "4202": 3538, + "419": 3539, + "416": 3540, + "402": 3541, + "403": 3542, + "458": 3543, + "468": 3544, + "459": 3545, + "4076": 3546, + "4116": 3547, + "515": 3548, + "593": 3549, + "596": 3550, + "587": 3551, + "558": 3552, + "5135": 3553, + "5663": 3554, + "5934": 3555, + "5924": 3556, + "601": 3557, + "657": 3558, + "656": 3559, + "693": 3560, + "696": 3561, + "653": 3562, + "76": 3563, + "78": 3564, + "706": 3565, + "703": 3566, + "757": 3567, + "792": 3568, + "787": 3569, + "733": 3570, + "768": 3571, + "732": 3572, + "735": 3573, + "769": 3574, + "7085": 3575, + "815": 3576, + "818": 3577, + "809": 3578, + "857": 3579, + "887": 3580, + "827": 3581, + "837": 3582, + "859": 3583, + "824": 3584, + "8075": 3585, + "8564": 3586, + "8076": 3587, + "8899": 3588, + "8125": 3589, + "910": 3590, + "905": 3591, + "909": 3592, + "957": 3593, + "979": 3594, + "9075": 3595, + "9142": 3596, + "Grab": 3597, + "Vis": 3598, + "Ġ18": 3599, + "Ġ115": 3600, + "Ġ112": 3601, + "Ġ123": 3602, + "Ġ24": 3603, + "ĠCan": 3604, + "1593": 3605, + "1568": 3606, + "1574": 3607, + "1524": 3608, + "1586": 3609, + "1548": 3610, + "Ġ34": 3611, + "2042": 3612, + "2073": 3613, + "2080": 3614, + "2070": 3615, + "1163": 3616, + "126": 3617, + "1268": 3618, + "1225": 3619, + "137": 3620, + "1319": 3621, + "1392": 3622, + "1395": 3623, + "144": 3624, + "145": 3625, + "147": 3626, + "1429": 3627, + "183": 3628, + "187": 3629, + "1818": 3630, + "1879": 3631, + "1867": 3632, + "1899": 3633, + "1964": 3634, + "1925": 3635, + "1612": 3636, + "1616": 3637, + "2220": 3638, + "2277": 3639, + "1714": 3640, + "Ġ72": 3641, + "Ġ94": 3642, + "Ġ95": 3643, + "Ġ41": 3644, + "Ġ125": 3645, + "Ġ82": 3646, + "Ġ83": 3647, + "Ġ85": 3648, + "5033": 3649, + "Ġ140": 3650, + "Ġchhote": 3651, + "6657": 3652, + "6631": 3653, + "6628": 3654, + "5714": 3655, + "5737": 3656, + "5752": 3657, + "5610": 3658, + "5657": 3659, + "5644": 3660, + "5658": 3661, + "5625": 3662, + "9389": 3663, + "9394": 3664, + "9610": 3665, + "9621": 3666, + "9690": 3667, + "9655": 3668, + "4414": 3669, + "4474": 3670, + "4465": 3671, + "926": 3672, + "9289": 3673, + "9242": 3674, + "9279": 3675, + "8921": 3676, + "8989": 3677, + "8775": 3678, + "273": 3679, + "2790": 3680, + "2785": 3681, + "4902": 3682, + "4998": 3683, + "4942": 3684, + "4978": 3685, + "4796": 3686, + "4764": 3687, + "4795": 3688, + "3302": 3689, + "3329": 3690, + "9821": 3691, + "9853": 3692, + "9868": 3693, + "9865": 3694, + "5313": 3695, + "5391": 3696, + "5324": 3697, + "5355": 3698, + "3718": 3699, + "3721": 3700, + "3764": 3701, + "4298": 3702, + "7200": 3703, + "7249": 3704, + "7291": 3705, + "5287": 3706, + "6818": 3707, + "6864": 3708, + "631": 3709, + "6305": 3710, + "7458": 3711, + "à¤Ĩत": 3712, + "6293": 3713, + "6244": 3714, + "6289": 3715, + "7772": 3716, + "7779": 3717, + "7775": 3718, + "3244": 3719, + "3286": 3720, + "3566": 3721, + "6510": 3722, + "7994": 3723, + "6944": 3724, + "6998": 3725, + "5901": 3726, + "5987": 3727, + "5947": 3728, + "Ġmentions": 3729, + "8823": 3730, + "8887": 3731, + "8881": 3732, + "4318": 3733, + "4347": 3734, + "4394": 3735, + "2922": 3736, + "2958": 3737, + "4018": 3738, + "4050": 3739, + "7549": 3740, + "5404": 3741, + "5456": 3742, + "6409": 3743, + "7392": 3744, + "7340": 3745, + "3823": 3746, + "8423": 3747, + "8480": 3748, + "6120": 3749, + "3907": 3750, + "9527": 3751, + "3668": 3752, + "4689": 3753, + "6744": 3754, + "2815": 3755, + "8675": 3756, + "9798": 3757, + "9785": 3758, + "7016": 3759, + "9915": 3760, + "6034": 3761, + "6039": 3762, + "8004": 3763, + "9961": 3764, + "9965": 3765, + "9969": 3766, + "1156": 3767, + "Ġwhose": 3768, + "2965": 3769, + "3872": 3770, + "3474": 3771, + "5184": 3772, + "7891": 3773, + "8032": 3774, + "8448": 3775, + "9660": 3776, + "4145": 3777, + "4146": 3778, + "4531": 3779, + "4536": 3780, + "6003": 3781, + "6009": 3782, + "6061": 3783, + "7227": 3784, + "9443": 3785, + "9449": 3786, + "9725": 3787, + "9728": 3788, + "Ġhona": 3789, + "3435": 3790, + "8312": 3791, + "2169": 3792, + "2666": 3793, + "2448": 3794, + "2697": 3795, + "3228": 3796, + "3064": 3797, + "3507": 3798, + "3043": 3799, + "3059": 3800, + "3577": 3801, + "3686": 3802, + "4195": 3803, + "4165": 3804, + "4029": 3805, + "4037": 3806, + "4586": 3807, + "4681": 3808, + "4599": 3809, + "5158": 3810, + "5939": 3811, + "5969": 3812, + "5875": 3813, + "5583": 3814, + "6574": 3815, + "6569": 3816, + "6935": 3817, + "6968": 3818, + "6534": 3819, + "7067": 3820, + "7034": 3821, + "7570": 3822, + "7929": 3823, + "7871": 3824, + "7336": 3825, + "7685": 3826, + "7326": 3827, + "7691": 3828, + "8151": 3829, + "8189": 3830, + "8094": 3831, + "8575": 3832, + "8873": 3833, + "8274": 3834, + "8378": 3835, + "8597": 3836, + "8241": 3837, + "9108": 3838, + "9057": 3839, + "9091": 3840, + "9576": 3841, + "9790": 3842, + "Visits": 3843, + "Saare": 3844, + "reation": 3845, + "mere": 3846, + "Ġamou": 3847, + "Ġcreation": 3848, + "Ġtwo": 3849, + "Ġexpens": 3850, + "Ġeve": 3851, + "à¥ĭ?": 3852, + "Ġthing": 3853, + "Ġwala": 3854, + "Ġstarted": 3855, + "Ġgaye": 3856, + "Ġboundaries": 3857, + "Ġkaunse": 3858, + "Ġworks": 3859, + "Ġamounts": 3860, + "Ġexpensive": 3861, + "Ġeven": 3862, + "Payment": 3863, + "sabhi": 3864, + "Ġcat": 3865, + "Ġski": 3866, + "Ġreal": 3867, + "Ġdept": 3868, + "ches": 3869, + "Ġstatuses": 3870, + "ĠBas": 3871, + "à¤ĩà¤ľ": 3872, + "Ġgoes": 3873, + "Accounts": 3874, + "Ġcatches": 3875, + "'m": 3876, + "Shipments": 3877, + "apes": 3878, + "dum": 3879, + "Ġche": 3880, + "Ġtal": 3881, + "Ġelect": 3882, + "Ġcreate": 3883, + "Ġcare": 3884, + "रफ": 3885, + "नà¤ĸ": 3886, + "archon": 3887, + "Ġà¤ħद": 3888, + "Ġतरफ": 3889, + "Ġतनà¤ĸ": 3890, + "Ġhoga": 3891, + "à¤ħर": 3892, + "Ġskip": 3893, + "apest": 3894, + "Ġcheapest": 3895, + "Ġtally": 3896, + "Ġelectronics": 3897, + "..": 3898, + "110": 3899, + "30": 3900, + "Dik": 3901, + "In": 3902, + "Kh": 3903, + "ko": 3904, + "so": 3905, + "ĺà¤Ł": 3906, + "Ġà¤ĺà¤Ł": 3907, + "Ġsubs": 3908, + "Ġbani": 3909, + "ĠMake": 3910, + "Ġkin": 3911, + "Ġà¤ķस": 3912, + "Ġà¤ķà¤ł": 3913, + "Ġdin": 3914, + "Ġdow": 3915, + "लà¤ķर": 3916, + "Ġcosts": 3917, + "Ġcurrently": 3918, + "Ġà¤ħम": 3919, + "Ġnulls": 3920, + "112": 3921, + "Ġasc": 3922, + "Ġपरव": 3923, + "123": 3924, + "140": 3925, + "Ġalso": 3926, + "Ġuska": 3927, + "Devices": 3928, + "ekdum": 3929, + "Ġbanaye": 3930, + "Ġà¤īतरत": 3931, + "à¤Łà¤®à¤°": 3932, + "Dikhao": 3933, + "Kharchon": 3934, + "Ġà¤ĺà¤Łà¤¤": 3935, + "Ġdown": 3936, + "Ek": 3937, + "Payments": 3938, + "Subscriptions": 3939, + "ppe": 3940, + "rked": 3941, + "tte": 3942, + "ybe": 3943, + "Ġquan": 3944, + "tities": 3945, + "Ġappe": 3946, + "upper": 3947, + "Ġsi": 3948, + "Ġsas": 3949, + "Ġwell": 3950, + "Ġear": 3951, + "Ġnorth": 3952, + "Ġdena": 3953, + "रप": 3954, + "à¤°à¤ľ": 3955, + "Ġसस": 3956, + "Ġà¤¸à¤Ł": 3957, + "Ġjaye": 3958, + "ĠNeed": 3959, + "Ġmarked": 3960, + "Ġmaybe": 3961, + "à¥ĭà¤Ĥ,": 3962, + "Ġunko": 3963, + "Ġà¤Ĩà¤ĸ": 3964, + "Ġtotals": 3965, + "Ġlena": 3966, + "Ġतब": 3967, + "Ġhisab": 3968, + "Ġà¤īपà¤ķरण": 3969, + "Ġyeah": 3970, + "Ġwritte": 3971, + "à¤īत": 3972, + "Ġignores": 3973, + "Ġsome": 3974, + "Ġà¤Ńर": 3975, + "Ġamong": 3976, + "Ġरà¤ĸà¤ķर": 3977, + "Ġaaye": 3978, + "Ġtimes": 3979, + "Ticket": 3980, + "à¤ĩनव": 3981, + "ĠBasically": 3982, + "Ġquantities": 3983, + "Ġappear": 3984, + "Ġwritten": 3985, + "Expenses": 3986, + "Sare": 3987, + "Shipment": 3988, + "TH": 3989, + "bounda": 3990, + "ix": 3991, + "jo": 3992, + "lhi": 3993, + "low": 3994, + "nly": 3995, + "ood": 3996, + "payment": 3997, + "sare": 3998, + "wa": 3999, + "à¦": 4000, + "peat": 4001, + "inta": 4002, + "invoice": 4003, + "ides": 4004, + "maine": 4005, + "Ġaa": 4006, + "Ġaana": 4007, + "Ġaise": 4008, + "|>'": 4009, + "Ġcould": 4010, + "ĠIt": 4011, + "Products": 4012, + "ump": 4013, + "ays": 4014, + "Ġsouth": 4015, + "Ġsides": 4016, + "Ġsays": 4017, + "Ġrepeat": 4018, + "Ġisnt": 4019, + "Ġmix": 4020, + "atever": 4021, + "Ġbalan": 4022, + "Ġem": 4023, + "ĠMujhe": 4024, + "Ġdelhi": 4025, + "okar": 4026, + "Ġfara": 4027, + "Ġdump": 4028, + "Ġसà¤ĸ": 4029, + "à¥Ģà¤ģ": 4030, + "Ġmehen": 4031, + "लत": 4032, + "मह": 4033, + "ĠOnly": 4034, + "Ġjab": 4035, + "पà¤ķरण": 4036, + "Ġनल": 4037, + "Ġmauj": 4038, + "Ġà¤īतर": 4039, + "à¤ļढ": 4040, + "Ġhokar": 4041, + "à¤ľà¤¬": 4042, + "Ġlana": 4043, + "Ġleke": 4044, + "Ġjinhe": 4045, + "Ġbaat": 4046, + "Ġbaaki": 4047, + "Ġफर": 4048, + "Ġjinkay": 4049, + "Ġpehli": 4050, + "Ġkaunsa": 4051, + "Ġlink": 4052, + "Ġà¤ĩसस": 4053, + "Ġuski": 4054, + "à¤īपà¤ķरण": 4055, + "Ġchinta": 4056, + "Ġanyone": 4057, + "Ġhuye": 4058, + "Ġdikhwa": 4059, + "Ġpuri": 4060, + "Ġpoore": 4061, + "Ġwhatever": 4062, + "Ġjinkii": 4063, + "kinds": 4064, + "Ġchoti": 4065, + "like": 4066, + "à¤Ĩप": 4067, + "Ġresult": 4068, + "ners": 4069, + "...": 4070, + "Invoices": 4071, + "Invoice": 4072, + "uppercase": 4073, + "Ġsasti": 4074, + "Ġearners": 4075, + "boundary": 4076, + "lower": 4077, + "Ġmixed": 4078, + "Ġbalances": 4079, + "Ġfarak": 4080, + "Ġmaujood": 4081 + }, + "merges": [ + [ + "\"", + ":" + ], + [ + ",", + "\"" + ], + [ + "\":", + "\"" + ], + [ + "{", + "\"" + ], + [ + "t", + "i" + ], + [ + "\"", + ",\"" + ], + [ + "e", + "s" + ], + [ + "p", + "e" + ], + [ + "r", + "o" + ], + [ + "o", + "n" + ], + [ + "r", + "e" + ], + [ + "\":", + "{\"" + ], + [ + "i", + "n" + ], + [ + "T", + "E" + ], + [ + "s", + "e" + ], + [ + "t", + "a" + ], + [ + "t", + "y" + ], + [ + "ti", + "on" + ], + [ + "q", + "u" + ], + [ + "h", + "e" + ], + [ + "}", + "}" + ], + [ + "à", + "¤" + ], + [ + "i", + "d" + ], + [ + "m", + "e" + ], + [ + "ta", + "b" + ], + [ + "e", + "c" + ], + [ + "ty", + "pe" + ], + [ + "a", + "l" + ], + [ + "s", + "t" + ], + [ + "p", + "ro" + ], + [ + "t", + "e" + ], + [ + "m", + "a" + ], + [ + "pe", + "r" + ], + [ + "ti", + "es" + ], + [ + "per", + "ties" + ], + [ + "Ġ", + "a" + ], + [ + "R", + "E" + ], + [ + "a", + "d" + ], + [ + "ec", + "t" + ], + [ + "j", + "ect" + ], + [ + "\":", + "[" + ], + [ + "c", + "he" + ], + [ + "che", + "ma" + ], + [ + "n", + "t" + ], + [ + "]", + ",\"" + ], + [ + "o", + "l" + ], + [ + "e", + "r" + ], + [ + "n", + "a" + ], + [ + "na", + "me" + ], + [ + "in", + "g" + ], + [ + "r", + "i" + ], + [ + "tab", + "l" + ], + [ + "<", + "|" + ], + [ + "|", + ">" + ], + [ + "Ġ", + "c" + ], + [ + "er", + "y" + ], + [ + "qu", + "ery" + ], + [ + "c", + "ri" + ], + [ + "L", + "E" + ], + [ + "i", + "s" + ], + [ + "a", + "se" + ], + [ + "re", + "d" + ], + [ + "qu", + "i" + ], + [ + "re", + "qui" + ], + [ + "requi", + "red" + ], + [ + "u", + "p" + ], + [ + "d", + "a" + ], + [ + "n", + "d" + ], + [ + "\"", + "}}" + ], + [ + "d", + "es" + ], + [ + "des", + "cri" + ], + [ + "X", + "T" + ], + [ + "RE", + "A" + ], + [ + "p", + "tion" + ], + [ + "Ġ", + "TE" + ], + [ + "ĠTE", + "XT" + ], + [ + "r", + "ing" + ], + [ + "st", + "ring" + ], + [ + "o", + "r" + ], + [ + "Ġ", + "I" + ], + [ + "E", + "R" + ], + [ + "tab", + "ase" + ], + [ + "da", + "tabase" + ], + [ + "\":[", + "\"" + ], + [ + "ĠI", + "N" + ], + [ + "descri", + "ption" + ], + [ + "G", + "ER" + ], + [ + "TE", + "GER" + ], + [ + "ĠIN", + "TEGER" + ], + [ + "g", + "ro" + ], + [ + "gro", + "up" + ], + [ + "\"", + "],\"" + ], + [ + "p", + "u" + ], + [ + "d", + "i" + ], + [ + ".", + "\",\"" + ], + [ + "pu", + "t" + ], + [ + "nt", + "s" + ], + [ + "o", + "b" + ], + [ + "f", + "al" + ], + [ + "P", + "ro" + ], + [ + "di", + "tion" + ], + [ + "S", + "chema" + ], + [ + "in", + "put" + ], + [ + "al", + "Pro" + ], + [ + "pro", + "perties" + ], + [ + "ad", + "dition" + ], + [ + "ob", + "ject" + ], + [ + "fal", + "se" + ], + [ + "input", + "Schema" + ], + [ + "alPro", + "perties" + ], + [ + "addition", + "alProperties" + ], + [ + "Ġ", + "t" + ], + [ + "a", + "r" + ], + [ + "q", + "l" + ], + [ + "s", + "ql" + ], + [ + "d", + "e" + ], + [ + "n", + "s" + ], + [ + "à", + "¥" + ], + [ + "Ġ", + "à¤" + ], + [ + "Ġ", + "P" + ], + [ + "o", + "ol" + ], + [ + "pro", + "ject" + ], + [ + ")", + ";" + ], + [ + "u", + "te" + ], + [ + "x", + "ec" + ], + [ + "xec", + "ute" + ], + [ + "a", + "c" + ], + [ + "Ġ", + "(" + ], + [ + "Ġ", + "T" + ], + [ + "i", + "st" + ], + [ + "A", + "B" + ], + [ + "C", + "REA" + ], + [ + "ĠT", + "AB" + ], + [ + "CREA", + "TE" + ], + [ + "ĠTAB", + "LE" + ], + [ + "ool", + "s" + ], + [ + "s", + "chema" + ], + [ + "tabl", + "es" + ], + [ + "l", + "y" + ], + [ + "tabl", + "e" + ], + [ + "me", + "nts" + ], + [ + "Ġ", + "database" + ], + [ + "on", + "ly" + ], + [ + "e", + "nd" + ], + [ + "|>", + "{\"" + ], + [ + "group", + "s" + ], + [ + "i", + "c" + ], + [ + "m", + "o" + ], + [ + ",", + "{\"" + ], + [ + "}}", + ",{\"" + ], + [ + "C", + "T" + ], + [ + "I", + "M" + ], + [ + "K", + "E" + ], + [ + "A", + "R" + ], + [ + "E", + "LE" + ], + [ + "ELE", + "CT" + ], + [ + "S", + "ELECT" + ], + [ + "f", + "or" + ], + [ + "R", + "IM" + ], + [ + "Ġ", + "KE" + ], + [ + "ĠP", + "RIM" + ], + [ + "AR", + "Y" + ], + [ + "ĠKE", + "Y" + ], + [ + "ĠPRIM", + "ARY" + ], + [ + "\"}}", + ",\"" + ], + [ + "u", + "m" + ], + [ + "\"", + "}" + ], + [ + "a", + "y" + ], + [ + "a", + "b" + ], + [ + "Ġ", + "s" + ], + [ + "Ġ", + "re" + ], + [ + "Ġa", + "nd" + ], + [ + "Ġc", + "ol" + ], + [ + "Ġcol", + "um" + ], + [ + "Ġ", + "in" + ], + [ + "Ġcolum", + "ns" + ], + [ + "a", + "t" + ], + [ + "Ġ", + "w" + ], + [ + "Ġt", + "he" + ], + [ + "Ġ", + "D" + ], + [ + "t", + "s" + ], + [ + "Ġ", + "pro" + ], + [ + "Ġa", + "c" + ], + [ + "E", + "RE" + ], + [ + "A", + "TE" + ], + [ + "Ġ", + "REA" + ], + [ + ");", + "\",\"" + ], + [ + "ĠD", + "ATE" + ], + [ + "ĠREA", + "L" + ], + [ + "Ġac", + "c" + ], + [ + "te", + "m" + ], + [ + "R", + "e" + ], + [ + "i", + "tem" + ], + [ + "de", + "mo" + ], + [ + "Ġ", + "is" + ], + [ + "e", + "xecute" + ], + [ + "Ġw", + "he" + ], + [ + "e", + "l" + ], + [ + "d", + "b" + ], + [ + "t", + "ools" + ], + [ + "Ġ", + "query" + ], + [ + "Ġ", + "A" + ], + [ + "x", + "t" + ], + [ + "e", + "d" + ], + [ + "te", + "xt" + ], + [ + "l", + "ist" + ], + [ + "Re", + "ad" + ], + [ + "schema", + "s" + ], + [ + "c", + "on" + ], + [ + "Ġ", + "l" + ], + [ + "\":[", + "{\"" + ], + [ + "Ġ", + "m" + ], + [ + "Ġ", + "tables" + ], + [ + "Ġre", + "ad" + ], + [ + "a", + "s" + ], + [ + "Q", + "L" + ], + [ + "S", + "QL" + ], + [ + "Ġ", + "SELECT" + ], + [ + "L", + "ist" + ], + [ + "i", + "r" + ], + [ + "item", + "s" + ], + [ + "v", + "id" + ], + [ + "es", + "s" + ], + [ + "Ġpro", + "vid" + ], + [ + "b", + "ac" + ], + [ + "bac", + "k" + ], + [ + "s", + "ing" + ], + [ + "u", + "se" + ], + [ + "s", + "k" + ], + [ + "Ġthe", + "ir" + ], + [ + "Ġprovid", + "ed" + ], + [ + "for", + "ma" + ], + [ + "Ġwhe", + "n" + ], + [ + "Ġin", + "forma" + ], + [ + "Ġinforma", + "tion" + ], + [ + "as", + "s" + ], + [ + "is", + "sing" + ], + [ + "Ġm", + "issing" + ], + [ + "Ġacc", + "ess" + ], + [ + "Ġ", + "U" + ], + [ + "ta", + "nt" + ], + [ + "E", + "xecute" + ], + [ + "p", + "ol" + ], + [ + "r", + "ay" + ], + [ + "}", + ",\"" + ], + [ + "Ġ", + "required" + ], + [ + "}}", + "],\"" + ], + [ + "is", + "tant" + ], + [ + "\"}}", + "},\"" + ], + [ + "Ġt", + "ools" + ], + [ + "ar", + "ray" + ], + [ + "ac", + "tion" + ], + [ + "ic", + "y" + ], + [ + "ĠA", + "sk" + ], + [ + "con", + "text" + ], + [ + "back", + "end" + ], + [ + "use", + "r" + ], + [ + "ass", + "istant" + ], + [ + "ĠU", + "se" + ], + [ + "pol", + "icy" + ], + [ + "o", + "u" + ], + [ + ".", + "\"}" + ], + [ + "al", + "l" + ], + [ + "ab", + "el" + ], + [ + "E", + "S" + ], + [ + "ar", + "g" + ], + [ + "c", + "all" + ], + [ + "u", + "ments" + ], + [ + "arg", + "uments" + ], + [ + "k", + "e" + ], + [ + "a", + "te" + ], + [ + "i", + "ty" + ], + [ + "Ġ", + "group" + ], + [ + "Ġl", + "abel" + ], + [ + "C", + "ES" + ], + [ + "F", + "ERE" + ], + [ + "N", + "CES" + ], + [ + "Ġ", + "RE" + ], + [ + ");", + "\"],\"" + ], + [ + "FERE", + "NCES" + ], + [ + "ĠRE", + "FERENCES" + ], + [ + "\":[", + "],\"" + ], + [ + "\"}}", + "<|" + ], + [ + ")", + ");\"],\"" + ], + [ + "R", + "O" + ], + [ + "à¤", + "¾" + ], + [ + "à¥", + "ĩ" + ], + [ + "ti", + "c" + ], + [ + "ic", + "es" + ], + [ + "Ġ", + "F" + ], + [ + "tic", + "ke" + ], + [ + "RO", + "M" + ], + [ + "ĠF", + "ROM" + ], + [ + "Ġ", + "for" + ], + [ + "or", + "d" + ], + [ + "Ġ", + "b" + ], + [ + ";", + "\"}}<|" + ], + [ + "Ġc", + "ity" + ], + [ + "o", + "st" + ], + [ + "p", + "l" + ], + [ + "Ġ", + "e" + ], + [ + "ord", + "er" + ], + [ + "re", + "ate" + ], + [ + "m", + "y" + ], + [ + "my", + "sql" + ], + [ + "reate", + "d" + ], + [ + "Ġc", + "reated" + ], + [ + "s", + "up" + ], + [ + "ab", + "ase" + ], + [ + "sup", + "abase" + ], + [ + "Ġ", + "p" + ], + [ + "mo", + "u" + ], + [ + "mou", + "nt" + ], + [ + "Ġ", + "M" + ], + [ + "Ġa", + "mount" + ], + [ + "\"}", + ",\"" + ], + [ + "à¤", + "Ĥ" + ], + [ + "u", + "n" + ], + [ + "Ġ", + "k" + ], + [ + "Ġ", + "n" + ], + [ + "a", + "n" + ], + [ + "Ġ", + "order" + ], + [ + "pe", + "c" + ], + [ + "Ġ", + "table" + ], + [ + "_", + "_" + ], + [ + "Ġ", + "on" + ], + [ + "pec", + "t" + ], + [ + "ns", + "pect" + ], + [ + "Ġ", + "de" + ], + [ + "b", + "e" + ], + [ + "descri", + "be" + ], + [ + "p", + "p" + ], + [ + "l", + "e" + ], + [ + "p", + "es" + ], + [ + "Ġ", + "ty" + ], + [ + "Ġty", + "pes" + ], + [ + "Ġà¤", + "ķ" + ], + [ + "h", + "i" + ], + [ + "Ġon", + "e" + ], + [ + "i", + "ts" + ], + [ + "I", + "nspect" + ], + [ + "da", + "te" + ], + [ + "g", + "re" + ], + [ + "y", + "SQL" + ], + [ + "ĠM", + "ySQL" + ], + [ + "ĠP", + "ost" + ], + [ + "gre", + "SQL" + ], + [ + "ĠPost", + "greSQL" + ], + [ + "Ġs", + "ta" + ], + [ + "o", + "k" + ], + [ + "à¥", + "į" + ], + [ + "o", + "ok" + ], + [ + "ook", + "s" + ], + [ + "Ġ", + "f" + ], + [ + "à¤", + "¿" + ], + [ + "y", + "e" + ], + [ + "Ġ", + "*" + ], + [ + "Ġa", + "u" + ], + [ + "o", + "t" + ], + [ + "u", + "c" + ], + [ + "v", + "is" + ], + [ + "vis", + "its" + ], + [ + "Ġn", + "ot" + ], + [ + "d", + "uc" + ], + [ + "duc", + "ts" + ], + [ + "o", + "in" + ], + [ + "à¤", + "°" + ], + [ + "Ġ", + "W" + ], + [ + "c", + "e" + ], + [ + "m", + "pl" + ], + [ + "ticke", + "ts" + ], + [ + "o", + "ye" + ], + [ + "mpl", + "oye" + ], + [ + "mploye", + "es" + ], + [ + "u", + "b" + ], + [ + "ption", + "s" + ], + [ + "ub", + "s" + ], + [ + "ubs", + "cri" + ], + [ + "v", + "ices" + ], + [ + "ay", + "ments" + ], + [ + "ubscri", + "ptions" + ], + [ + "v", + "o" + ], + [ + "r", + "s" + ], + [ + "vo", + "ices" + ], + [ + "ou", + "rs" + ], + [ + "ours", + "es" + ], + [ + "H", + "ERE" + ], + [ + "ĠW", + "HERE" + ], + [ + "hi", + "p" + ], + [ + "hip", + "ments" + ], + [ + "u", + "de" + ], + [ + "ou", + "nts" + ], + [ + "Ġau", + "x" + ], + [ + "Ġnot", + "es" + ], + [ + "ticke", + "t" + ], + [ + "st", + "ude" + ], + [ + "stude", + "nts" + ], + [ + "t", + "ments" + ], + [ + "pp", + "oin" + ], + [ + "ppoin", + "tments" + ], + [ + "e", + "t" + ], + [ + "pe", + "ns" + ], + [ + "x", + "pens" + ], + [ + "xpens", + "es" + ], + [ + "i", + "t" + ], + [ + "u", + "s" + ], + [ + "Ġ", + "d" + ], + [ + "à¤", + "ķ" + ], + [ + "ro", + "w" + ], + [ + "Ġre", + "c" + ], + [ + "r", + "un" + ], + [ + "Ġb", + "ooks" + ], + [ + "Ġ", + "visits" + ], + [ + "Ġà¤", + "¸" + ], + [ + "Ġ", + "row" + ], + [ + "Ġ", + "tickets" + ], + [ + "Ġpro", + "ducts" + ], + [ + "Ġp", + "ayments" + ], + [ + "Ġe", + "mployees" + ], + [ + "Ġde", + "vices" + ], + [ + "Ġs", + "ubscriptions" + ], + [ + "Ġc", + "ourses" + ], + [ + "Ġin", + "voices" + ], + [ + "h", + "a" + ], + [ + "or", + "y" + ], + [ + "at", + "h" + ], + [ + "ro", + "m" + ], + [ + "Ġf", + "rom" + ], + [ + "Ġs", + "hipments" + ], + [ + "or", + "de" + ], + [ + "c", + "p" + ], + [ + "m", + "cp" + ], + [ + "à¥", + "Ģ" + ], + [ + "à¥ĩ", + "à¤Ĥ" + ], + [ + "Ġorder", + "s" + ], + [ + "Ġ", + "students" + ], + [ + "Ġrow", + "s" + ], + [ + "à¤", + "¤" + ], + [ + "in", + "d" + ], + [ + "0", + "0" + ], + [ + "Ġa", + "ppointments" + ], + [ + "orde", + "d" + ], + [ + "Ġacc", + "ounts" + ], + [ + "Ġc", + "u" + ], + [ + "Ġe", + "xpenses" + ], + [ + "Ġrec", + "orded" + ], + [ + "r", + "t" + ], + [ + "g", + "i" + ], + [ + "Ġsta", + "rt" + ], + [ + "à¥", + "¤" + ], + [ + "à¤", + "¨" + ], + [ + "c", + "h" + ], + [ + "is", + "pl" + ], + [ + "ispl", + "ay" + ], + [ + "t", + "us" + ], + [ + "Ġk", + "ind" + ], + [ + "ar", + "t" + ], + [ + "Ġsta", + "tus" + ], + [ + "me", + "nt" + ], + [ + "gi", + "on" + ], + [ + "p", + "art" + ], + [ + "part", + "ment" + ], + [ + "Ġre", + "gion" + ], + [ + "Ġde", + "partment" + ], + [ + "ate", + "g" + ], + [ + "ateg", + "ory" + ], + [ + "Ġc", + "ategory" + ], + [ + "g", + "et" + ], + [ + "à¤", + "¯" + ], + [ + "ti", + "n" + ], + [ + "o", + "re" + ], + [ + "tin", + "g" + ], + [ + "Ġ", + "1" + ], + [ + "v", + "e" + ], + [ + "o", + "c" + ], + [ + "r", + "a" + ], + [ + "Ġ", + "=" + ], + [ + "Ġ", + "se" + ], + [ + "ti", + "t" + ], + [ + "tit", + "le" + ], + [ + "Ġ", + "me" + ], + [ + "he", + "r" + ], + [ + "Ġà¤", + "ª" + ], + [ + "ra", + "ting" + ], + [ + "c", + "ore" + ], + [ + "Ġ", + "B" + ], + [ + "st", + "o" + ], + [ + "i", + "le" + ], + [ + "sto", + "me" + ], + [ + "stome", + "r" + ], + [ + "Ġà¤", + "®" + ], + [ + "Ġd", + "isplay" + ], + [ + "ri", + "ce" + ], + [ + "al", + "an" + ], + [ + "e", + "at" + ], + [ + "eat", + "her" + ], + [ + "alan", + "ce" + ], + [ + "Ġ", + "title" + ], + [ + "o", + "w" + ], + [ + "ar", + "ch" + ], + [ + "à¤", + "²" + ], + [ + "Ġs", + "core" + ], + [ + "Ġ", + "rating" + ], + [ + "Ġ", + "item" + ], + [ + "Ġc", + "ost" + ], + [ + "ti", + "ty" + ], + [ + "c", + "ity" + ], + [ + "qu", + "an" + ], + [ + "quan", + "tity" + ], + [ + "un", + "it" + ], + [ + "ar", + "y" + ], + [ + "al", + "ary" + ], + [ + "Ġà¤", + "¹" + ], + [ + "p", + "ath" + ], + [ + "Ġ", + "description" + ], + [ + "Ġp", + "rice" + ], + [ + "Ġ", + "ha" + ], + [ + "Ġb", + "alance" + ], + [ + "Ġcu", + "stomer" + ], + [ + "Ġ", + "name" + ], + [ + "Ġ", + "quantity" + ], + [ + "Ġs", + "alary" + ], + [ + "Ġwhe", + "re" + ], + [ + "à¥", + "ĭ" + ], + [ + "Ġ", + "'" + ], + [ + "à¤", + "®" + ], + [ + "a", + "h" + ], + [ + "ĠB", + "Y" + ], + [ + "à¥", + "Ī" + ], + [ + "à¤", + "ģ" + ], + [ + "Ġ", + "2" + ], + [ + "Ġk", + "a" + ], + [ + "Ġ", + "O" + ], + [ + "à¤", + "¹" + ], + [ + "Ġt", + "o" + ], + [ + "a", + "pp" + ], + [ + "\"", + "]" + ], + [ + "\"]", + "}}" + ], + [ + "N", + "T" + ], + [ + "or", + "t" + ], + [ + "à¤", + "ı" + ], + [ + "à¤", + "¸" + ], + [ + "Ġb", + "y" + ], + [ + "Ġ", + "5" + ], + [ + "b", + "ooks" + ], + [ + "Ġ", + "j" + ], + [ + "pro", + "ducts" + ], + [ + "Ġha", + "i" + ], + [ + "Ġ", + ">" + ], + [ + "e", + "mployees" + ], + [ + "s", + "hipments" + ], + [ + "in", + "voices" + ], + [ + "oc", + "s" + ], + [ + "ा", + "à¤ģ" + ], + [ + "s", + "ubscriptions" + ], + [ + "de", + "vices" + ], + [ + "c", + "ourses" + ], + [ + "Ġ", + "L" + ], + [ + "c", + "ounts" + ], + [ + "ac", + "counts" + ], + [ + "a", + "ppointments" + ], + [ + "Ġ", + "C" + ], + [ + "p", + "ayments" + ], + [ + "order", + "s" + ], + [ + "I", + "N" + ], + [ + "O", + "U" + ], + [ + "(", + "*" + ], + [ + "ĠC", + "OU" + ], + [ + "(*", + ")" + ], + [ + "ĠCOU", + "NT" + ], + [ + "'", + ";\"}}<|" + ], + [ + "d", + "ocs" + ], + [ + "e", + "xpenses" + ], + [ + "Ġà¤", + "ľ" + ], + [ + "Ġ2", + "0" + ], + [ + "à¤", + "Ł" + ], + [ + "t", + "ri" + ], + [ + "e", + "ve" + ], + [ + "tri", + "eve" + ], + [ + "re", + "n" + ], + [ + "s", + "i" + ], + [ + "G", + "et" + ], + [ + "se", + "arch" + ], + [ + "f", + "ile" + ], + [ + "re", + "ad" + ], + [ + "el", + "si" + ], + [ + "elsi", + "us" + ], + [ + "w", + "eather" + ], + [ + "he", + "it" + ], + [ + "ah", + "ren" + ], + [ + "ahren", + "heit" + ], + [ + "Re", + "trieve" + ], + [ + "à¤", + "ª" + ], + [ + "Ġà¤", + "¦" + ], + [ + "Ġf", + "ile" + ], + [ + "Ġ", + "ticket" + ], + [ + "e", + "n" + ], + [ + "à¤", + "ĸ" + ], + [ + "c", + "elsius" + ], + [ + "Ġw", + "eather" + ], + [ + "f", + "ahrenheit" + ], + [ + "h", + "ow" + ], + [ + "D", + "ER" + ], + [ + "R", + "DER" + ], + [ + "ĠO", + "RDER" + ], + [ + "à¤", + "¬" + ], + [ + "p", + "ort" + ], + [ + "Ġ", + "N" + ], + [ + "oc", + "u" + ], + [ + "i", + "l" + ], + [ + "Ġà¤", + "µ" + ], + [ + "n", + "ta" + ], + [ + "me", + "nta" + ], + [ + "ocu", + "menta" + ], + [ + "ocumenta", + "tion" + ], + [ + "Ġs", + "up" + ], + [ + "Ġsup", + "port" + ], + [ + "l", + "abel" + ], + [ + "Ġ", + "ke" + ], + [ + "Ġd", + "ocumentation" + ], + [ + "Ġ", + "v" + ], + [ + "es", + "t" + ], + [ + "u", + "e" + ], + [ + "re", + "nt" + ], + [ + "Ġ", + "its" + ], + [ + "à¥ĩà¤Ĥ", + "।" + ], + [ + "Ġ", + "text" + ], + [ + "S", + "e" + ], + [ + "r", + "rent" + ], + [ + "Ġcu", + "rrent" + ], + [ + "Se", + "arch" + ], + [ + "ĠI", + "D" + ], + [ + "Ġp", + "ath" + ], + [ + "\"]}}", + ",\"" + ], + [ + "en", + "um" + ], + [ + "Ġà¤ķ", + "र" + ], + [ + "c", + "l" + ], + [ + "Ġme", + "in" + ], + [ + "Ġt", + "h" + ], + [ + "Ġk", + "i" + ], + [ + "Ġv", + "al" + ], + [ + "à¥Ī", + "।" + ], + [ + "a", + "ri" + ], + [ + "Ġà¤", + "¨" + ], + [ + "l", + "l" + ], + [ + "Ġka", + "ro" + ], + [ + "f", + "y" + ], + [ + "Ġ", + "y" + ], + [ + "Ġ", + "o" + ], + [ + "Ġ", + "G" + ], + [ + "U", + "P" + ], + [ + "RO", + "UP" + ], + [ + "ĠG", + "ROUP" + ], + [ + "Ġ1", + "00" + ], + [ + "\"}", + "<|" + ], + [ + "à¥", + "Ĥ" + ], + [ + "Ġ", + "S" + ], + [ + "Ġ20", + "2" + ], + [ + "Ġ", + "di" + ], + [ + "Ġ", + "ma" + ], + [ + "Ġo", + "f" + ], + [ + "Ġà¤", + "²" + ], + [ + "l", + "ic" + ], + [ + "à¥", + "ģ" + ], + [ + "?", + "\"}<|" + ], + [ + "es", + "tion" + ], + [ + "qu", + "estion" + ], + [ + "cl", + "ari" + ], + [ + "clari", + "fy" + ], + [ + "b", + "lic" + ], + [ + "pu", + "blic" + ], + [ + "\"]}}", + "<|" + ], + [ + "ou", + "nt" + ], + [ + "Ġrec", + "ord" + ], + [ + "um", + "b" + ], + [ + "Ġdi", + "k" + ], + [ + "ou", + "t" + ], + [ + "Ġval", + "ue" + ], + [ + "à¤", + "¼" + ], + [ + "1", + "5" + ], + [ + "Ġ", + "all" + ], + [ + "Ġdik", + "ha" + ], + [ + "a", + "i" + ], + [ + "Ġn", + "e" + ], + [ + "Ġdikha", + "o" + ], + [ + "Ġrecord", + "s" + ], + [ + "ah", + "an" + ], + [ + "Ġà¤ľ", + "ह" + ], + [ + "Ġy", + "ou" + ], + [ + "Ġcolum", + "n" + ], + [ + "à¤", + "§" + ], + [ + "à¤", + "¦" + ], + [ + "à¤", + "¡" + ], + [ + "Ġà¤", + "ī" + ], + [ + "Ġà¤", + "ħ" + ], + [ + "Ġj", + "ahan" + ], + [ + "Ġ1", + "0" + ], + [ + "Ġ>", + "=" + ], + [ + "à¤", + "ļ" + ], + [ + "Ġà¤", + "Ķ" + ], + [ + "Ġ", + "schema" + ], + [ + "Ġ", + "da" + ], + [ + "Ġda", + "ta" + ], + [ + "u", + "ll" + ], + [ + "Ġ", + "J" + ], + [ + "à¥ĭ", + "à¤Ĥ" + ], + [ + "Ġe", + "nt" + ], + [ + "n", + "ow" + ], + [ + "Ġ", + "h" + ], + [ + "Ġ", + "3" + ], + [ + "ri", + "es" + ], + [ + "umb", + "ai" + ], + [ + "U", + "M" + ], + [ + "ĠS", + "UM" + ], + [ + "Ġent", + "ries" + ], + [ + "el", + "hi" + ], + [ + "'", + ");" + ], + [ + "');", + "\"}}<|" + ], + [ + "O", + "W" + ], + [ + "ĠL", + "OW" + ], + [ + "ĠLOW", + "ER" + ], + [ + "N", + "D" + ], + [ + "ĠA", + "ND" + ], + [ + "L", + "L" + ], + [ + "U", + "LL" + ], + [ + "ĠI", + "S" + ], + [ + "ĠN", + "ULL" + ], + [ + "Ġne", + "ed" + ], + [ + "Ġ5", + "00" + ], + [ + "Ġ", + "<" + ], + [ + "Ġà¤", + "¬" + ], + [ + "ĠD", + "ES" + ], + [ + "ĠDES", + "C" + ], + [ + "à¤", + "µ" + ], + [ + "S", + "C" + ], + [ + "ĠA", + "SC" + ], + [ + "O", + "IN" + ], + [ + "ĠO", + "N" + ], + [ + "ĠJ", + "OIN" + ], + [ + "I", + "T" + ], + [ + "IM", + "IT" + ], + [ + "ĠL", + "IMIT" + ], + [ + "ti", + "ve" + ], + [ + "Ġà¤Ķ", + "र" + ], + [ + "le", + "ase" + ], + [ + "k", + "now" + ], + [ + "Ġ", + "un" + ], + [ + "l", + "ect" + ], + [ + "te", + "nt" + ], + [ + "P", + "lease" + ], + [ + "à¥", + "ī" + ], + [ + "K", + "T" + ], + [ + "]", + "}" + ], + [ + "h", + "ist" + ], + [ + "t", + "ool" + ], + [ + "ro", + "le" + ], + [ + "con", + "tent" + ], + [ + "hist", + "ory" + ], + [ + "Ġl", + "i" + ], + [ + "Ġn", + "ull" + ], + [ + "Ġ", + "E" + ], + [ + "h", + "at" + ], + [ + "ta", + "l" + ], + [ + "Ġau", + "r" + ], + [ + "it", + "h" + ], + [ + "2", + "0" + ], + [ + "Ġw", + "ith" + ], + [ + "Ġà¤", + "Ĩ" + ], + [ + "out", + "h" + ], + [ + "Ġs", + "a" + ], + [ + "Ġto", + "tal" + ], + [ + "pe", + "nd" + ], + [ + "pend", + "ing" + ], + [ + "Ġl", + "ow" + ], + [ + "Ġw", + "o" + ], + [ + "ort", + "h" + ], + [ + "n", + "ic" + ], + [ + "ro", + "nic" + ], + [ + "ronic", + "s" + ], + [ + "un", + "e" + ], + [ + "lect", + "ronics" + ], + [ + "à¤", + "ĩ" + ], + [ + "y", + "a" + ], + [ + "Ġc", + "ount" + ], + [ + "c", + "u" + ], + [ + "à¤Ĥ", + "।" + ], + [ + "à¥", + "Į" + ], + [ + "Ġth", + "an" + ], + [ + "Re", + "il" + ], + [ + "Reil", + "ly" + ], + [ + "Ġk", + "o" + ], + [ + "et", + "ch" + ], + [ + "know", + "n" + ], + [ + "S", + "how" + ], + [ + "à¤", + "ľ" + ], + [ + "प", + "य" + ], + [ + "1", + "0" + ], + [ + "Ġ", + "na" + ], + [ + "Ġà¤", + "°" + ], + [ + "Ġ", + "hi" + ], + [ + "à¥", + "ĥ" + ], + [ + "Ġ", + "up" + ], + [ + "u", + "r" + ], + [ + "à¥Ī", + "à¤Ĥ" + ], + [ + "Ġd", + "o" + ], + [ + "Ġhi", + "g" + ], + [ + "à¤", + "Ń" + ], + [ + "Ġ", + "list" + ], + [ + "Ġà¤", + "ļ" + ], + [ + "Ġ", + "or" + ], + [ + "Ġ", + "pl" + ], + [ + "Ġस", + "à¤Ń" + ], + [ + "Ġà¤", + "¡" + ], + [ + "Ġà¤", + "¯" + ], + [ + "Ġs", + "how" + ], + [ + "ic", + "h" + ], + [ + "qu", + "al" + ], + [ + "Ġe", + "qual" + ], + [ + "à¤", + "¶" + ], + [ + "e", + "ase" + ], + [ + "Ġa", + "t" + ], + [ + "te", + "r" + ], + [ + "Ġa", + "v" + ], + [ + "Ġl", + "a" + ], + [ + "Ġha", + "ve" + ], + [ + "Ġj", + "oin" + ], + [ + "Ġ5", + "0" + ], + [ + "Ġhig", + "h" + ], + [ + "Ġk", + "ar" + ], + [ + "Ġre", + "t" + ], + [ + "Ġlow", + "est" + ], + [ + "D", + "elhi" + ], + [ + "W", + "hat" + ], + [ + ")", + ");\",\"" + ], + [ + "M", + "umbai" + ], + [ + "Ġth", + "at" + ], + [ + "Ġc", + "on" + ], + [ + "Ġ", + "u" + ], + [ + "Ġt", + "ool" + ], + [ + "ur", + "n" + ], + [ + "Ġret", + "urn" + ], + [ + "1", + "1" + ], + [ + "20", + "2" + ], + [ + "Ġ", + "H" + ], + [ + "Ġse", + "e" + ], + [ + "h", + "ich" + ], + [ + "d", + "isplay" + ], + [ + "ad", + "a" + ], + [ + "l", + "d" + ], + [ + "ou", + "ld" + ], + [ + "त", + "म" + ], + [ + "ब", + "स" + ], + [ + "र", + "म" + ], + [ + "ĠM", + "IN" + ], + [ + "à¥Ģ", + "à¤Ĥ" + ], + [ + "न", + "à¤ķ" + ], + [ + "ĠN", + "O" + ], + [ + "ĠNO", + "T" + ], + [ + "à¤", + "·" + ], + [ + "'", + "," + ], + [ + "V", + "G" + ], + [ + "ĠA", + "VG" + ], + [ + "a", + "il" + ], + [ + "Ġ<", + "=" + ], + [ + "Ġha", + "in" + ], + [ + "Ġ(", + "'" + ], + [ + "Ġ100", + "0" + ], + [ + "%", + "');\"}}<|" + ], + [ + "'", + "%" + ], + [ + "(", + "'%" + ], + [ + "E", + "E" + ], + [ + "E", + "T" + ], + [ + "I", + "KE" + ], + [ + "W", + "EE" + ], + [ + "ĠB", + "ET" + ], + [ + "ĠL", + "IKE" + ], + [ + "WEE", + "N" + ], + [ + "ĠBET", + "WEEN" + ], + [ + "g", + "e" + ], + [ + "Ġ100", + "00" + ], + [ + "E", + "AR" + ], + [ + "Y", + "EAR" + ], + [ + "A", + "V" + ], + [ + "IN", + "G" + ], + [ + "ĠH", + "AV" + ], + [ + "ĠHAV", + "ING" + ], + [ + "Ġ2", + "00" + ], + [ + "I", + "S" + ], + [ + "T", + "IN" + ], + [ + "ĠD", + "IS" + ], + [ + "TIN", + "CT" + ], + [ + "ĠDIS", + "TINCT" + ], + [ + "Ġp", + "re" + ], + [ + "v", + "i" + ], + [ + "Ġna", + "hi" + ], + [ + "U", + "n" + ], + [ + "Ġma", + "t" + ], + [ + "i", + "k" + ], + [ + "Ġ1", + "5" + ], + [ + "A", + "X" + ], + [ + "ĠM", + "AX" + ], + [ + "Ġw", + "al" + ], + [ + "Ġhigh", + "est" + ], + [ + "Ġन", + "ह" + ], + [ + "ou", + "s" + ], + [ + "Ġpre", + "vi" + ], + [ + "Ġprevi", + "ous" + ], + [ + "Ġn", + "ik" + ], + [ + "Ġà¤", + "Ł" + ], + [ + "c", + "ount" + ], + [ + "Ġl", + "e" + ], + [ + "ro", + "r" + ], + [ + "A", + "CT" + ], + [ + "R", + "ACT" + ], + [ + "XT", + "RACT" + ], + [ + "ĠE", + "XTRACT" + ], + [ + "O", + "NT" + ], + [ + "ONT", + "H" + ], + [ + "Ġà¤", + "¤" + ], + [ + "Ġw", + "a" + ], + [ + "cu", + "stomer" + ], + [ + "w", + "er" + ], + [ + "ns", + "wer" + ], + [ + "E", + "r" + ], + [ + "à¥Ī", + "?" + ], + [ + "ĠI", + "nspect" + ], + [ + "à¤", + "£" + ], + [ + "Un", + "known" + ], + [ + "ĠM", + "umbai" + ], + [ + "à¤", + "Ī" + ], + [ + "Ġb", + "e" + ], + [ + "C", + "an" + ], + [ + "\":\"", + "/" + ], + [ + "Ġpl", + "ease" + ], + [ + "u", + "st" + ], + [ + "Ġa", + "s" + ], + [ + "a", + "nswer" + ], + [ + "}}", + "]}" + ], + [ + "Ġ", + "g" + ], + [ + "ĠD", + "elhi" + ], + [ + "T", + "KT" + ], + [ + "ĠT", + "KT" + ], + [ + "T", + "he" + ], + [ + "Ġ500", + "0" + ], + [ + "Ġ", + "z" + ], + [ + "r", + "ue" + ], + [ + "t", + "rue" + ], + [ + "is", + "Er" + ], + [ + ".\"}", + "]}" + ], + [ + "isEr", + "ror" + ], + [ + "Ġc", + "ha" + ], + [ + "ac", + "h" + ], + [ + "Ġe", + "ach" + ], + [ + "Ġnik", + "al" + ], + [ + "ac", + "tive" + ], + [ + "Ġ", + "0" + ], + [ + "Ġà¤ħ", + "ध" + ], + [ + "Ġ15", + "00" + ], + [ + "à¤", + "Ĺ" + ], + [ + "Ġ", + "mo" + ], + [ + "u", + "t" + ], + [ + "S", + "outh" + ], + [ + "u", + "es" + ], + [ + "Ġval", + "ues" + ], + [ + "N", + "orth" + ], + [ + "pu", + "ll" + ], + [ + "P", + "une" + ], + [ + "à¤ģ", + "।" + ], + [ + "B", + "ooks" + ], + [ + "E", + "lectronics" + ], + [ + "hi", + "ye" + ], + [ + "Ġcha", + "hiye" + ], + [ + "Ġà¤ķ", + "म" + ], + [ + "c", + "end" + ], + [ + "Ġy", + "a" + ], + [ + "cend", + "ing" + ], + [ + "ĠS", + "outh" + ], + [ + "Ġreturn", + "ed" + ], + [ + "c", + "an" + ], + [ + "f", + "i" + ], + [ + "à¤", + "¢" + ], + [ + "Ġac", + "tive" + ], + [ + "Ġ", + "SQL" + ], + [ + "y", + "ada" + ], + [ + "Ġz", + "yada" + ], + [ + "Ġ", + "pending" + ], + [ + "Ġcon", + "n" + ], + [ + "ĠE", + "lectronics" + ], + [ + "ĠN", + "orth" + ], + [ + "ा", + "à¤Ĥ" + ], + [ + "ĠP", + "une" + ], + [ + "r", + "es" + ], + [ + "Ġà¤", + "Ĺ" + ], + [ + "Ġconn", + "ec" + ], + [ + "Ġप", + "र" + ], + [ + "Ġto", + "p" + ], + [ + "i", + "m" + ], + [ + "'", + "'" + ], + [ + "n", + "y" + ], + [ + "s", + "how" + ], + [ + "Ġà¤Ĩ", + "प" + ], + [ + "a", + "mount" + ], + [ + "Ġk", + "ya" + ], + [ + "à¥Ī", + "," + ], + [ + "Ġ", + "groups" + ], + [ + "Ġस", + "बस" + ], + [ + "Ġla", + "o" + ], + [ + "1", + "2" + ], + [ + "ĠB", + "ooks" + ], + [ + "t", + "h" + ], + [ + "Ġ", + "i" + ], + [ + "im", + "um" + ], + [ + "Ġb", + "est" + ], + [ + "F", + "etch" + ], + [ + "c", + "ost" + ], + [ + "Ġ", + "/" + ], + [ + "D", + "isplay" + ], + [ + "Ġin", + "cl" + ], + [ + "il", + "ter" + ], + [ + "à¤", + "Ŀ" + ], + [ + "i", + "ve" + ], + [ + "Ġs", + "pec" + ], + [ + "Ġspec", + "i" + ], + [ + "Ġ", + "out" + ], + [ + "Ġconnec", + "tion" + ], + [ + "n", + "e" + ], + [ + "़", + "ा" + ], + [ + "in", + "e" + ], + [ + "u", + "j" + ], + [ + "uj", + "he" + ], + [ + "Ġli", + "ye" + ], + [ + "Ġm", + "in" + ], + [ + "Ġ", + "items" + ], + [ + "b", + "alance" + ], + [ + ".\"}", + "<|" + ], + [ + "ल", + "म" + ], + [ + "Ġnikal", + "o" + ], + [ + "Ġà¤ī", + "न" + ], + [ + "Ġwa", + "nt" + ], + [ + "Ġli", + "ke" + ], + [ + "Ġe", + "x" + ], + [ + "a", + "ge" + ], + [ + "s", + "core" + ], + [ + "Ġ", + "pe" + ], + [ + "f", + "ore" + ], + [ + "à¤ĩ", + "à¤ı" + ], + [ + "te", + "d" + ], + [ + "on", + "th" + ], + [ + "gi", + "ve" + ], + [ + "Ġm", + "onth" + ], + [ + "v", + "ery" + ], + [ + "Ġe", + "very" + ], + [ + "k", + "h" + ], + [ + "Ġmat", + "ch" + ], + [ + "Ġa", + "re" + ], + [ + "Ġc", + "ase" + ], + [ + "ab", + "le" + ], + [ + "Ġma", + "ny" + ], + [ + "Ġs", + "h" + ], + [ + "ल", + "ब" + ], + [ + "Ġka", + "m" + ], + [ + "Ġj", + "in" + ], + [ + "Ġs", + "or" + ], + [ + "p", + "rice" + ], + [ + "र", + "द" + ], + [ + "ब", + "द" + ], + [ + "s", + "alary" + ], + [ + "se", + "t" + ], + [ + "Ġ202", + "4" + ], + [ + "Ġk", + "it" + ], + [ + "Ġwal", + "i" + ], + [ + "Ġme", + "an" + ], + [ + "Ġ202", + "6" + ], + [ + "er", + "age" + ], + [ + "Ġav", + "erage" + ], + [ + "a", + "re" + ], + [ + "Ġav", + "ail" + ], + [ + "Ġsor", + "ted" + ], + [ + "Ġb", + "a" + ], + [ + "Ġbe", + "fore" + ], + [ + "Ġà¤", + "ĸ" + ], + [ + "1", + "3" + ], + [ + "Ġmo", + "re" + ], + [ + "Ġj", + "ust" + ], + [ + "Ġavail", + "able" + ], + [ + "umb", + "er" + ], + [ + "Ġw", + "hich" + ], + [ + "c", + "k" + ], + [ + "e", + "nt" + ], + [ + "Ġ", + "pull" + ], + [ + "à¥Īà¤Ĥ", + "?" + ], + [ + "r", + "y" + ], + [ + "Ġun", + "known" + ], + [ + "Ġà¤", + "«" + ], + [ + "Ġ", + "YEAR" + ], + [ + "1", + "4" + ], + [ + "Ġh", + "is" + ], + [ + "a", + "ab" + ], + [ + "Ġhis", + "aab" + ], + [ + "W", + "hich" + ], + [ + "M", + "ujhe" + ], + [ + "Ġn", + "umber" + ], + [ + "Ġkar", + "ke" + ], + [ + "Ġsa", + "are" + ], + [ + "r", + "an" + ], + [ + "Ġ202", + "2" + ], + [ + "1", + "8" + ], + [ + "त", + "न" + ], + [ + "Ġà¤", + "¶" + ], + [ + "ir", + "st" + ], + [ + "Ġf", + "irst" + ], + [ + "Ġdo", + "es" + ], + [ + "F", + "ind" + ], + [ + "न", + "तम" + ], + [ + "Ġsa", + "ari" + ], + [ + "res", + "ent" + ], + [ + "Ġà¤ī", + "स" + ], + [ + "D", + "M" + ], + [ + "m", + "d" + ], + [ + "REA", + "DM" + ], + [ + "READM", + "E" + ], + [ + "u", + "ns" + ], + [ + "Ġ202", + "3" + ], + [ + "c", + "t" + ], + [ + "Ġप", + "ह" + ], + [ + "n", + "i" + ], + [ + "Ġ", + "get" + ], + [ + "à¤", + "Ľ" + ], + [ + "1", + "9" + ], + [ + "p", + "or" + ], + [ + "1", + "6" + ], + [ + "M", + "ONTH" + ], + [ + "c", + "s" + ], + [ + "s", + "al" + ], + [ + "re", + "por" + ], + [ + "cs", + "v" + ], + [ + "sal", + "es" + ], + [ + "repor", + "ts" + ], + [ + "O", + "ut" + ], + [ + "Out", + "put" + ], + [ + "2", + "2" + ], + [ + "Ġ", + "st" + ], + [ + "Ġस", + "म" + ], + [ + "1", + "7" + ], + [ + "Ġsh", + "ould" + ], + [ + "st", + "e" + ], + [ + "Ġth", + "is" + ], + [ + "Ġ", + "K" + ], + [ + "T", + "h" + ], + [ + "Ġin", + "ste" + ], + [ + "ĠW", + "ould" + ], + [ + "Th", + "is" + ], + [ + "Ġinste", + "ad" + ], + [ + "Ġà¤Ĩप", + "à¤ķ" + ], + [ + "Ġjin", + "k" + ], + [ + "Ġप", + "ढ" + ], + [ + "Ġs", + "ab" + ], + [ + "ब", + "ल" + ], + [ + "2", + "1" + ], + [ + "Ġf", + "ilter" + ], + [ + "ĠM", + "ONTH" + ], + [ + "रद", + "र" + ], + [ + "2", + "3" + ], + [ + "Ġma", + "x" + ], + [ + "Ġmin", + "imum" + ], + [ + "Ġc", + "ounts" + ], + [ + "Ġ", + "6" + ], + [ + "a", + "st" + ], + [ + "Ġf", + "etch" + ], + [ + "reate", + "r" + ], + [ + "Ġ202", + "5" + ], + [ + "Ġg", + "reater" + ], + [ + "Ġas", + "cending" + ], + [ + "Ġc", + "elsius" + ], + [ + "Ġl", + "ess" + ], + [ + "Ġf", + "ahrenheit" + ], + [ + "Ġf", + "ail" + ], + [ + "Ġपह", + "ल" + ], + [ + "Ġà¤ħ", + "न" + ], + [ + "र", + "ण" + ], + [ + "t", + "he" + ], + [ + "Ġpe", + "h" + ], + [ + "Ġpeh", + "le" + ], + [ + "Ġà¤", + "ı" + ], + [ + "pu", + "r" + ], + [ + "s", + "on" + ], + [ + "uc", + "know" + ], + [ + "Ġ", + "7" + ], + [ + "Ġ", + "it" + ], + [ + "G", + "ive" + ], + [ + "n", + "ot" + ], + [ + "t", + "xt" + ], + [ + "set", + "up" + ], + [ + "not", + "es" + ], + [ + "à¤", + "ĵ" + ], + [ + "fi", + "ed" + ], + [ + "Ġspeci", + "fied" + ], + [ + "ण", + "न" + ], + [ + "Ġà¤Ĺ", + "णन" + ], + [ + "k", + "i" + ], + [ + "ai", + "pur" + ], + [ + "Ġब", + "ड" + ], + [ + "Ġequal", + "s" + ], + [ + "Ġà¤īस", + "स" + ], + [ + "m", + "all" + ], + [ + "Ġs", + "mall" + ], + [ + "Ġjoin", + "s" + ], + [ + "Ġब", + "त" + ], + [ + "s", + "ta" + ], + [ + "j", + "son" + ], + [ + "con", + "fi" + ], + [ + "confi", + "g" + ], + [ + "Ġmatch", + "es" + ], + [ + "Ġ", + "des" + ], + [ + "Ġà¤Ĩ", + "ध" + ], + [ + "Ġh", + "o" + ], + [ + "a", + "tion" + ], + [ + "à¤", + "¥" + ], + [ + "Ġkit", + "ni" + ], + [ + "che", + "ck" + ], + [ + "Ġwal", + "e" + ], + [ + "v", + "el" + ], + [ + "ri", + "ty" + ], + [ + "Ġse", + "cu" + ], + [ + "Ġà¤ī", + "प" + ], + [ + "Ġle", + "vel" + ], + [ + "Ġsecu", + "rity" + ], + [ + "Ġà¤īप", + "लब" + ], + [ + "f", + "f" + ], + [ + "Ġjink", + "a" + ], + [ + "Ġ", + "only" + ], + [ + "Ġb", + "ut" + ], + [ + "Ġj", + "a" + ], + [ + "Ġ", + "ye" + ], + [ + "Ġp", + "resent" + ], + [ + "Ġle", + "ast" + ], + [ + "Ġdes", + "cending" + ], + [ + "à¤ĸ", + "न" + ], + [ + "h", + "o" + ], + [ + "Ġ", + "back" + ], + [ + "Ġma", + "ine" + ], + [ + "Ġsab", + "se" + ], + [ + "Ġw", + "ri" + ], + [ + "P", + "I" + ], + [ + "n", + "tic" + ], + [ + "w", + "hat" + ], + [ + "Ġau", + "the" + ], + [ + "ntic", + "ation" + ], + [ + "Ġauthe", + "ntication" + ], + [ + "ta", + "il" + ], + [ + "Ġde", + "tail" + ], + [ + "Ġdetail", + "s" + ], + [ + "Ġt", + "ry" + ], + [ + "न", + "म" + ], + [ + "à¥ĩà¤Ĥ", + "," + ], + [ + "Ġ", + "check" + ], + [ + "Ġs", + "ort" + ], + [ + "pl", + "ease" + ], + [ + "up", + "s" + ], + [ + "Ġback", + "ups" + ], + [ + "Ġn", + "o" + ], + [ + "Ġd", + "is" + ], + [ + "व", + "ल" + ], + [ + "Ġsmall", + "est" + ], + [ + "Ġa", + "b" + ], + [ + "x", + "es" + ], + [ + "de", + "xes" + ], + [ + "o", + "m" + ], + [ + "a", + "p" + ], + [ + "Ġa", + "g" + ], + [ + "Ġpl", + "s" + ], + [ + "à¤ķ", + "र" + ], + [ + "l", + "ate" + ], + [ + "Ġb", + "ot" + ], + [ + "Ġh", + "as" + ], + [ + "à¤", + "ħ" + ], + [ + "à¤", + "ŀ" + ], + [ + "Ġ", + "9" + ], + [ + "Ġf", + "i" + ], + [ + "Ġà¤ī", + "à¤ļ" + ], + [ + "श", + "न" + ], + [ + "Ġà¤ī", + "नà¤ķ" + ], + [ + "Ġu", + "sing" + ], + [ + "Ġla", + "st" + ], + [ + "à¤ļ", + "तम" + ], + [ + "te", + "ll" + ], + [ + "L", + "ucknow" + ], + [ + "à¤ķ", + "त" + ], + [ + "on", + "e" + ], + [ + "Ġf", + "ind" + ], + [ + "Ġprovid", + "e" + ], + [ + "Ġp", + "ar" + ], + [ + "Ġ", + "give" + ], + [ + "Ġ", + "4" + ], + [ + "Ġwith", + "in" + ], + [ + "tin", + "ct" + ], + [ + "Ġdis", + "tinct" + ], + [ + "Ġ", + "orde" + ], + [ + "Ġorde", + "red" + ], + [ + "f", + "etch" + ], + [ + "J", + "aipur" + ], + [ + "Ġà¤", + "Ĭ" + ], + [ + "sta", + "rt" + ], + [ + "el", + "d" + ], + [ + "Ġh", + "ar" + ], + [ + "Ġfi", + "eld" + ], + [ + "Ġko", + "i" + ], + [ + "Ġye", + "ar" + ], + [ + "a", + "m" + ], + [ + "a", + "in" + ], + [ + "u", + "re" + ], + [ + "Ġ1", + "2" + ], + [ + "Ġmax", + "imum" + ], + [ + "Ġस", + "à¤ķत" + ], + [ + "us", + "ive" + ], + [ + "Ġka", + "uns" + ], + [ + "Ġout", + "put" + ], + [ + "Ġst", + "u" + ], + [ + "Ġkauns", + "i" + ], + [ + "m", + "es" + ], + [ + "Ġna", + "mes" + ], + [ + "Ġv", + "i" + ], + [ + "ou", + "nd" + ], + [ + "Ġb", + "as" + ], + [ + "Ġh", + "one" + ], + [ + "à¤ĩ", + "à¤Ł" + ], + [ + "0", + "6" + ], + [ + "Q", + "u" + ], + [ + "c", + "reated" + ], + [ + "ar", + "a" + ], + [ + "Qu", + "ery" + ], + [ + "al", + "ly" + ], + [ + "ac", + "t" + ], + [ + "a", + "ta" + ], + [ + "Ġincl", + "ude" + ], + [ + "प", + "र" + ], + [ + "ran", + "ge" + ], + [ + "Ġà¤Ĭ", + "पर" + ], + [ + "'", + "t" + ], + [ + "o", + "ve" + ], + [ + "Ġab", + "ove" + ], + [ + "à¤ķ", + "à¤Ł" + ], + [ + "Ġà¤", + "ĩ" + ], + [ + "Ġ", + "how" + ], + [ + "Ġह", + "र" + ], + [ + "Ġd", + "ocs" + ], + [ + "c", + "orded" + ], + [ + "re", + "corded" + ], + [ + "Ġम", + "ह" + ], + [ + "Ġ202", + "1" + ], + [ + "Ġl", + "in" + ], + [ + "ब", + "र" + ], + [ + "Ġब", + "र" + ], + [ + "Ġse", + "t" + ], + [ + "Ġà¤ı", + "à¤ķ" + ], + [ + "l", + "ab" + ], + [ + "Ġmat", + "lab" + ], + [ + "i", + "qu" + ], + [ + "Ġun", + "iqu" + ], + [ + "Ġuniqu", + "e" + ], + [ + "स", + "त" + ], + [ + "Ġà¤Ķ", + "सत" + ], + [ + "Ġmo", + "st" + ], + [ + "Ġthe", + "m" + ], + [ + "ce", + "ed" + ], + [ + "Ġex", + "ceed" + ], + [ + "Ġ", + "us" + ], + [ + "Ġ", + "al" + ], + [ + "Ġf", + "ound" + ], + [ + "Ġincl", + "usive" + ], + [ + "Ġstu", + "ff" + ], + [ + "Ġ", + "8" + ], + [ + "Ġ", + "tell" + ], + [ + "Ġà¤ħ", + "à¤ļ" + ], + [ + "Ġfail", + "ed" + ], + [ + "Ġl", + "arg" + ], + [ + "P", + "ull" + ], + [ + "Ġà¤ĩ", + "स" + ], + [ + "Ġ", + "ta" + ], + [ + "al", + "cu" + ], + [ + "alcu", + "late" + ], + [ + "Ġja", + "ha" + ], + [ + "e", + "en" + ], + [ + "w", + "een" + ], + [ + "Ġb", + "et" + ], + [ + "202", + "6" + ], + [ + "Ġbet", + "ween" + ], + [ + "Ġu", + "sk" + ], + [ + "Ġa", + "r" + ], + [ + "़", + "ि" + ], + [ + "Ġusk", + "e" + ], + [ + "Ġà¤ķर", + "न" + ], + [ + "Ġba", + "ta" + ], + [ + "Ġb", + "ada" + ], + [ + "Ġp", + "ad" + ], + [ + "Ġà¤ķ", + "न" + ], + [ + "य", + "ह" + ], + [ + "स", + "बस" + ], + [ + "Ġag", + "ain" + ], + [ + "त", + "लब" + ], + [ + "Ġम", + "तलब" + ], + [ + "Ġw", + "hat" + ], + [ + "Ġkar", + "ne" + ], + [ + "Ġ1", + "1" + ], + [ + "Ġj", + "o" + ], + [ + "0", + "2" + ], + [ + "à¤ĩ", + "ल" + ], + [ + "202", + "4" + ], + [ + "Ġspeci", + "fy" + ], + [ + "Ġk", + "now" + ], + [ + "B", + "est" + ], + [ + "a", + "uns" + ], + [ + "ĠK", + "auns" + ], + [ + "ĠKauns", + "a" + ], + [ + "Ġव", + "ह" + ], + [ + "0", + "8" + ], + [ + "Ġe", + "it" + ], + [ + "Ġeit", + "her" + ], + [ + "v", + "er" + ], + [ + "स", + "म" + ], + [ + "Ġus", + "se" + ], + [ + "202", + "2" + ], + [ + "a", + "o" + ], + [ + "à¤Ł", + "र" + ], + [ + "à¤Ĥ।", + "\"}<|" + ], + [ + "T", + "ool" + ], + [ + "e", + "w" + ], + [ + "e", + "h" + ], + [ + "Y", + "eh" + ], + [ + "us", + "am" + ], + [ + "Ġma", + "usam" + ], + [ + "Ġup", + "ar" + ], + [ + "ĠK", + "ya" + ], + [ + "ar", + "ab" + ], + [ + "Ġb", + "arab" + ], + [ + "Ġbarab", + "ar" + ], + [ + "d", + "ing" + ], + [ + "Ġf", + "il" + ], + [ + "Ġlow", + "er" + ], + [ + "ter", + "m" + ], + [ + "Ġ", + "sql" + ], + [ + "kh", + "ne" + ], + [ + "5", + "0" + ], + [ + "Ġli", + "khne" + ], + [ + "Ġc", + "an" + ], + [ + "a", + "use" + ], + [ + "र", + "व" + ], + [ + "Ġo", + "ver" + ], + [ + "Ġfil", + "te" + ], + [ + "Ġl", + "ook" + ], + [ + "Ġse", + "lect" + ], + [ + "Ġब", + "ढ" + ], + [ + "ns", + "i" + ], + [ + "रम", + "बद" + ], + [ + "à¤", + "«" + ], + [ + "Ġà¤ħ", + "à¤ľ" + ], + [ + "o", + "se" + ], + [ + "Ġsta", + "r" + ], + [ + "202", + "3" + ], + [ + "0", + "1" + ], + [ + "H", + "ow" + ], + [ + "202", + "5" + ], + [ + "Ġa", + "n" + ], + [ + "Ġs", + "um" + ], + [ + "़", + "à¥Ģ" + ], + [ + "Ġi", + "f" + ], + [ + "Ġstar", + "ting" + ], + [ + "à¤", + "ī" + ], + [ + "Ġse", + "arch" + ], + [ + "Ġjoin", + "ing" + ], + [ + "Ġin", + "s" + ], + [ + "Ġde", + "kh" + ], + [ + "Ġwri", + "ting" + ], + [ + "Ġun", + "ki" + ], + [ + "term", + "ine" + ], + [ + "Ġ1", + "3" + ], + [ + "ob", + "ara" + ], + [ + "t", + "om" + ], + [ + "Ġon", + "es" + ], + [ + "Ġbot", + "tom" + ], + [ + "u", + "ding" + ], + [ + "Ġw", + "e" + ], + [ + "|>", + "/" + ], + [ + "ा", + "।" + ], + [ + "0", + "4" + ], + [ + "j", + "ust" + ], + [ + "Ġup", + "per" + ], + [ + "e", + "te" + ], + [ + "Ġà¤Ĩ", + "à¤ĩà¤Ł" + ], + [ + "Ġà¤Ĩà¤ĩà¤Ł", + "म" + ], + [ + "v", + "ing" + ], + [ + "Ġfilte", + "red" + ], + [ + "Ġ", + "ra" + ], + [ + "uc", + "h" + ], + [ + "F", + "ilter" + ], + [ + "à¤ĩ", + "स" + ], + [ + "Ġc", + "h" + ], + [ + "Ġth", + "ose" + ], + [ + "'", + "s" + ], + [ + "k", + "ed" + ], + [ + "Ġb", + "ase" + ], + [ + "Ġd", + "on" + ], + [ + "Ġà¤", + "Ľ" + ], + [ + "ह", + "त" + ], + [ + "A", + "PI" + ], + [ + "ĠA", + "PI" + ], + [ + "Ġun", + "he" + ], + [ + "à¤ĩ", + "प" + ], + [ + "g", + "n" + ], + [ + "र", + "त" + ], + [ + "Ġ", + "id" + ], + [ + "Ġi", + "gn" + ], + [ + "Ġ", + "res" + ], + [ + "Ġdata", + "set" + ], + [ + "ic", + "ally" + ], + [ + "Ġs", + "o" + ], + [ + "Ġb", + "ad" + ], + [ + "a", + "g" + ], + [ + "Ġlarg", + "est" + ], + [ + "Ġd", + "h" + ], + [ + "ec", + "ause" + ], + [ + "Ġb", + "ecause" + ], + [ + "Ġta", + "k" + ], + [ + "j", + "ah" + ], + [ + "l", + "a" + ], + [ + "Ġwa", + "jah" + ], + [ + "Ġba", + "ad" + ], + [ + "el", + "ect" + ], + [ + "Ġincl", + "uding" + ], + [ + "0", + "5" + ], + [ + "in", + "dexes" + ], + [ + "Ġin", + "dexes" + ], + [ + "S", + "elect" + ], + [ + "nd", + "er" + ], + [ + "Ġhig", + "her" + ], + [ + "Ġu", + "nder" + ], + [ + "à¤ķ", + "तम" + ], + [ + "ल", + "à¤Ĺ" + ], + [ + "B", + "ring" + ], + [ + "Ġत", + "à¤ķ" + ], + [ + "Ġli", + "m" + ], + [ + "0", + "3" + ], + [ + "Ġwri", + "te" + ], + [ + "Ġbase", + "d" + ], + [ + "f", + "ind" + ], + [ + "à¤ī", + "न" + ], + [ + "0", + "9" + ], + [ + "ri", + "ct" + ], + [ + "Ġst", + "rict" + ], + [ + "Ġstrict", + "ly" + ], + [ + "ne", + "ed" + ], + [ + "Ġthe", + "re" + ], + [ + "Ġ", + "per" + ], + [ + "K", + "ri" + ], + [ + "p", + "ya" + ], + [ + "Kri", + "pya" + ], + [ + "P", + "resent" + ], + [ + "र", + "स" + ], + [ + "र", + "य" + ], + [ + "f", + "ter" + ], + [ + "Ġf", + "ull" + ], + [ + "ĠD", + "obara" + ], + [ + "Ġlin", + "es" + ], + [ + "ti", + "fy" + ], + [ + "D", + "e" + ], + [ + "0", + "7" + ], + [ + "Ġa", + "ap" + ], + [ + "Ġà¤ħ", + "à¤ķ" + ], + [ + "ाà¤Ĥ", + "," + ], + [ + "Ġc", + "o" + ], + [ + "De", + "termine" + ], + [ + "un", + "ki" + ], + [ + "mpl", + "ete" + ], + [ + "ष", + "र" + ], + [ + "Ġco", + "mplete" + ], + [ + "y", + "unki" + ], + [ + "Ġ", + "date" + ], + [ + "ta", + "in" + ], + [ + "ate", + "d" + ], + [ + "Ġe", + "k" + ], + [ + "à¥Ĥ", + "à¤Ĥ" + ], + [ + "Ġcon", + "tain" + ], + [ + "़", + "à¥ĩà¤Ĥ।" + ], + [ + "Ġsa", + "ath" + ], + [ + "he", + "ck" + ], + [ + "à¤Ľ", + "ल" + ], + [ + "Ġvi", + "ew" + ], + [ + "Ġà¤", + "¥" + ], + [ + "Ġ", + "ok" + ], + [ + "ur", + "u" + ], + [ + "Ġsh", + "uru" + ], + [ + "on", + "g" + ], + [ + "Ġl", + "o" + ], + [ + "Ġha", + "ving" + ], + [ + "Ġex", + "act" + ], + [ + "Ġ", + "range" + ], + [ + "Ġa", + "ny" + ], + [ + "Ġय", + "ह" + ], + [ + "Ġkit", + "na" + ], + [ + "Ġid", + "en" + ], + [ + "k", + "in" + ], + [ + "Ġle", + "kin" + ], + [ + "ata", + "o" + ], + [ + "K", + "ya" + ], + [ + "i", + "ye" + ], + [ + "Ġa", + "fter" + ], + [ + "Ġex", + "ist" + ], + [ + "Ġ1", + "4" + ], + [ + "Ġn", + "ow" + ], + [ + "Ġsa", + "me" + ], + [ + "Ġbas", + "ically" + ], + [ + "Ġbata", + "o" + ], + [ + "Ġj", + "u" + ], + [ + "I", + "s" + ], + [ + "Ġc", + "ap" + ], + [ + "E", + "xt" + ], + [ + "r", + "act" + ], + [ + "Ġs", + "ure" + ], + [ + "Ext", + "ract" + ], + [ + "i", + "g" + ], + [ + "Ġb", + "ig" + ], + [ + "Ġvi", + "a" + ], + [ + "Ġar", + "range" + ], + [ + "D", + "a" + ], + [ + "k", + "te" + ], + [ + "r", + "in" + ], + [ + "Ġsa", + "kte" + ], + [ + "Ġbe", + "ing" + ], + [ + "Da", + "tabase" + ], + [ + "rin", + "t" + ], + [ + "Ġdi", + "ff" + ], + [ + "c", + "ase" + ], + [ + "Ġbot", + "h" + ], + [ + "à¥ĭ", + "।" + ], + [ + "Ġgive", + "n" + ], + [ + "व", + "रण" + ], + [ + "Ġmatch", + "ing" + ], + [ + "Ġd", + "id" + ], + [ + "म", + "ल" + ], + [ + "de", + "partment" + ], + [ + "ran", + "ked" + ], + [ + "Ġwe", + "re" + ], + [ + "u", + "nd" + ], + [ + "Ġpad", + "ho" + ], + [ + "nsi", + "tive" + ], + [ + "Ġdh", + "und" + ], + [ + "in", + "ce" + ], + [ + "़", + "à¥ĩ" + ], + [ + "़", + "à¥ĭà¤Ĥ" + ], + [ + "m", + "p" + ], + [ + "त", + "र" + ], + [ + "Ġà¤ķर", + "à¤ķ" + ], + [ + "à¥Īà¤Ĥ", + "," + ], + [ + "फ", + "ल" + ], + [ + "mp", + "ty" + ], + [ + "à¥Ģ", + "।" + ], + [ + "Ġkar", + "ta" + ], + [ + "Ġbas", + "is" + ], + [ + "ĠL", + "ucknow" + ], + [ + "C", + "ould" + ], + [ + "c", + "ategory" + ], + [ + "़", + "à¥į" + ], + [ + "ran", + "g" + ], + [ + "Ġar", + "rang" + ], + [ + "e", + "k" + ], + [ + "ec", + "h" + ], + [ + "ĠJ", + "aipur" + ], + [ + "Ġbe", + "ech" + ], + [ + "P", + "ich" + ], + [ + "Ġ", + "fal" + ], + [ + "Ġfal", + "l" + ], + [ + "C", + "alculate" + ], + [ + "Ġdi", + "ya" + ], + [ + "Ġc", + "alculate" + ], + [ + "Ġfall", + "s" + ], + [ + "ty", + "ek" + ], + [ + "Ġp", + "ra" + ], + [ + "oin", + "ts" + ], + [ + "ĠH", + "ow" + ], + [ + "Ġpra", + "tyek" + ], + [ + "he", + "y" + ], + [ + "Ġव", + "र" + ], + [ + "Ġh", + "u" + ], + [ + "न", + "त" + ], + [ + "Ġ", + "er" + ], + [ + "Ġto", + "h" + ], + [ + "Ġiden", + "tify" + ], + [ + "i", + "z" + ], + [ + "द", + "स" + ], + [ + "Ġद", + "स" + ], + [ + "Ġब", + "न" + ], + [ + "Ġà¤Ĺ", + "à¤Ī" + ], + [ + "in", + "ti" + ], + [ + "Ġg", + "inti" + ], + [ + "Ġal", + "ong" + ], + [ + "Ġg", + "ay" + ], + [ + "Ġl", + "oc" + ], + [ + "Ġm", + "uch" + ], + [ + "Ġp", + "a" + ], + [ + "Ġjoin", + "ed" + ], + [ + "d", + "s" + ], + [ + "g", + "h" + ], + [ + "Ġre", + "trieve" + ], + [ + "1", + "00" + ], + [ + "Ġ", + "run" + ], + [ + "a", + "ting" + ], + [ + "Ġà¤", + "¢" + ], + [ + "Ġह", + "à¤Ł" + ], + [ + "Ġdiff", + "e" + ], + [ + "Ġsab", + "hi" + ], + [ + "P", + "rint" + ], + [ + "pe", + "d" + ], + [ + "up", + "lic" + ], + [ + "Ġb", + "el" + ], + [ + "Ġbad", + "h" + ], + [ + "f", + "o" + ], + [ + "Ġin", + "fo" + ], + [ + "Ġbadh", + "te" + ], + [ + "Ġ", + "end" + ], + [ + "Ġw", + "as" + ], + [ + "Ġexceed", + "s" + ], + [ + "Ġal", + "ag" + ], + [ + "Ġch", + "ho" + ], + [ + "Ġh", + "ue" + ], + [ + "Ġhave", + "n" + ], + [ + "Ġ", + "ranked" + ], + [ + "à¤ķ", + "रण" + ], + [ + "à¤", + "ĥ" + ], + [ + "Ġdik", + "h" + ], + [ + "Ġma", + "ke" + ], + [ + "Ġla", + "ao" + ], + [ + "Ġra", + "kh" + ], + [ + "Ġexceed", + "ing" + ], + [ + "Ġà¤", + "Ń" + ], + [ + "Ġba", + "na" + ], + [ + "Ġrakh", + "te" + ], + [ + "Ġà¤Ĺ", + "य" + ], + [ + "Ġign", + "ore" + ], + [ + "Ġd", + "ue" + ], + [ + "Ġja", + "an" + ], + [ + "a", + "al" + ], + [ + "Ġnik", + "aal" + ], + [ + "se", + "nsitive" + ], + [ + "Ġlook", + "ing" + ], + [ + "Ġbana", + "o" + ], + [ + "Ġarrang", + "ed" + ], + [ + "Ġस", + "र" + ], + [ + "L", + "ook" + ], + [ + "k", + "s" + ], + [ + "sup", + "port" + ], + [ + "Ġwo", + "h" + ], + [ + "Pich", + "la" + ], + [ + "6", + "6" + ], + [ + "ĠP", + "lease" + ], + [ + "pec", + "tive" + ], + [ + "p", + "t" + ], + [ + "Ġinclude", + "d" + ], + [ + "Ġल", + "à¤Ĺ" + ], + [ + "Ġa", + "m" + ], + [ + "स", + "प" + ], + [ + "Ġर", + "à¤ĸ" + ], + [ + "Ġ6", + "0" + ], + [ + "sta", + "tus" + ], + [ + "Ġ", + "pu" + ], + [ + "id", + "e" + ], + [ + "Ġb", + "hi" + ], + [ + "Ġins", + "ide" + ], + [ + "Ġc", + "le" + ], + [ + "Ġp", + "oints" + ], + [ + "Ġde", + "le" + ], + [ + "Ġ10", + "3" + ], + [ + "Ġà¤īस", + "à¤ķ" + ], + [ + "Ġdekh", + "o" + ], + [ + "Ġpu", + "ra" + ], + [ + "m", + "ujhe" + ], + [ + "tem", + "pt" + ], + [ + "ir", + "f" + ], + [ + "Ġat", + "tempt" + ], + [ + "à¥Ī", + "à¤Ĥ।" + ], + [ + "C", + "heck" + ], + [ + "L", + "ast" + ], + [ + "Ġ1", + "6" + ], + [ + "Ġju", + "da" + ], + [ + "re", + "trieve" + ], + [ + "k", + "ing" + ], + [ + "Ġ", + "sto" + ], + [ + "re", + "ating" + ], + [ + "Ġs", + "an" + ], + [ + "kh", + "ya" + ], + [ + "Ġsto", + "red" + ], + [ + "Ġsan", + "khya" + ], + [ + "Ġer", + "ror" + ], + [ + "h", + "ing" + ], + [ + "it", + "ni" + ], + [ + "Ġfield", + "s" + ], + [ + "Ġk", + "yunki" + ], + [ + "Ġwith", + "out" + ], + [ + "Ġlim", + "its" + ], + [ + "th", + "ing" + ], + [ + "M", + "a" + ], + [ + "Ġà¤ļ", + "ल" + ], + [ + "Ġtry", + "ing" + ], + [ + "Ġbig", + "g" + ], + [ + "Ma", + "ine" + ], + [ + "Ġt", + "ha" + ], + [ + "Ġà¤ħ", + "लà¤Ĺ" + ], + [ + "J", + "ust" + ], + [ + "re", + "gion" + ], + [ + "à¤ħ", + "लà¤Ĺ" + ], + [ + "Ġ6", + "00" + ], + [ + "5", + "7" + ], + [ + "g", + "ar" + ], + [ + "qu", + "en" + ], + [ + "Ġse", + "quen" + ], + [ + "Ġexact", + "ly" + ], + [ + "Ġsequen", + "ce" + ], + [ + "i", + "ated" + ], + [ + "s", + "oc" + ], + [ + "v", + "ity" + ], + [ + "ti", + "vity" + ], + [ + "Ġp", + "o" + ], + [ + "Ġse", + "nsi" + ], + [ + "Ġas", + "soc" + ], + [ + "Ġsensi", + "tivity" + ], + [ + "Ġassoc", + "iated" + ], + [ + "o", + "ring" + ], + [ + "र", + "à¤ķ" + ], + [ + "Ġà¤Ĩ", + "à¤ķ" + ], + [ + "à¤ľ", + "ह" + ], + [ + "Ġg", + "o" + ], + [ + "Ġevery", + "thing" + ], + [ + "Ġdon", + "t" + ], + [ + "Ġsa", + "ara" + ], + [ + "Ġlarg", + "er" + ], + [ + "Ġign", + "oring" + ], + [ + "Ġlim", + "it" + ], + [ + "फल", + "त" + ], + [ + "Ġins", + "pect" + ], + [ + "l", + "ind" + ], + [ + "à¥Ĥ", + "à¤ģ।" + ], + [ + "व", + "ह" + ], + [ + "Ġe", + "ns" + ], + [ + "Ġgay", + "a" + ], + [ + "Ġend", + "s" + ], + [ + "à¤Ĥ", + "," + ], + [ + "à¤ģ", + "," + ], + [ + "Ġdid", + "n" + ], + [ + "y", + "an" + ], + [ + "Ġ", + "ot" + ], + [ + "Ġc", + "om" + ], + [ + "Ġres", + "pective" + ], + [ + "Ġot", + "her" + ], + [ + "Ġw", + "h" + ], + [ + "Ġ5", + "5" + ], + [ + "W", + "o" + ], + [ + "te", + "nts" + ], + [ + "Ġà¤ķर", + "त" + ], + [ + "Ġcon", + "tents" + ], + [ + "Ġjin", + "ke" + ], + [ + "Ġ", + "el" + ], + [ + "à¤", + "ł" + ], + [ + "al", + "c" + ], + [ + "ol", + "e" + ], + [ + "Ġc", + "alc" + ], + [ + "Ġwh", + "ole" + ], + [ + "Ġel", + "se" + ], + [ + "C", + "ount" + ], + [ + "Ġ", + "use" + ], + [ + "Ġw", + "ahan" + ], + [ + "Ġl", + "ate" + ], + [ + "Ġok", + "ay" + ], + [ + "Ġlate", + "r" + ], + [ + "ta", + "rt" + ], + [ + "as", + "ing" + ], + [ + "ि", + "à¤Ĥ" + ], + [ + "c", + "re" + ], + [ + "Ġin", + "cre" + ], + [ + "Ġmat", + "ter" + ], + [ + "Ġover", + "all" + ], + [ + "Ġincre", + "asing" + ], + [ + "Ġun", + "ke" + ], + [ + "Ġà¤Ĩ", + "र" + ], + [ + "al", + "ag" + ], + [ + "Pro", + "vid" + ], + [ + "Ġbel", + "ow" + ], + [ + "Provid", + "e" + ], + [ + "Ġent", + "ry" + ], + [ + "late", + "d" + ], + [ + "5", + "6" + ], + [ + "D", + "ocumentation" + ], + [ + "Ġthe", + "n" + ], + [ + "9", + "3" + ], + [ + "9", + "6" + ], + [ + "g", + "ate" + ], + [ + "gre", + "gate" + ], + [ + "Ġag", + "gregate" + ], + [ + "Ġhu", + "i" + ], + [ + "4", + "4" + ], + [ + "p", + "as" + ], + [ + "Ġcon", + "tent" + ], + [ + "Ġwa", + "pas" + ], + [ + "9", + "2" + ], + [ + "Ġs", + "uch" + ], + [ + "Ġe", + "mpty" + ], + [ + "ra", + "nd" + ], + [ + "Ġg", + "rand" + ], + [ + "in", + "sensitive" + ], + [ + "Ġa", + "ay" + ], + [ + "Ġa", + "age" + ], + [ + "प", + "स" + ], + [ + "Ġex", + "cl" + ], + [ + "Ġchho", + "ti" + ], + [ + "Ġ", + "db" + ], + [ + "Ġp", + "ata" + ], + [ + "Ġpad", + "h" + ], + [ + "B", + "atao" + ], + [ + "Ġre", + "mo" + ], + [ + "Ġद", + "र" + ], + [ + "Ġà¤Ĩ", + "à¤Ĺ" + ], + [ + "ĠF", + "ind" + ], + [ + "Ġaay", + "in" + ], + [ + "8", + "9" + ], + [ + "à¤Ĥ", + "?" + ], + [ + "i", + "pe" + ], + [ + "Ġà¤ħ", + "व" + ], + [ + "Ġsa", + "al" + ], + [ + "Ġdhund", + "o" + ], + [ + "B", + "e" + ], + [ + "a", + "ro" + ], + [ + "Ġs", + "ince" + ], + [ + "Ġz", + "aro" + ], + [ + "Ġjink", + "i" + ], + [ + "Be", + "fore" + ], + [ + "Ġn", + "on" + ], + [ + "Ġप", + "त" + ], + [ + "Ġ16", + "00" + ], + [ + "Ġर", + "ह" + ], + [ + "Ġcontain", + "s" + ], + [ + "D", + "ocs" + ], + [ + "Ġ", + "[" + ], + [ + "Ġm", + "ujhe" + ], + [ + "à¥ĩ", + "।" + ], + [ + "Ġthan", + "ks" + ], + [ + "Ġkit", + "ne" + ], + [ + "Ġdele", + "te" + ], + [ + "n", + "da" + ], + [ + "pu", + "te" + ], + [ + "ou", + "nda" + ], + [ + "un", + "o" + ], + [ + "Ġkar", + "na" + ], + [ + "Ġch", + "uno" + ], + [ + "Ġरà¤ĸ", + "त" + ], + [ + "Ġs", + "ha" + ], + [ + "Ġ1", + "10" + ], + [ + "om", + "pute" + ], + [ + "Ġlower", + "case" + ], + [ + "8", + "7" + ], + [ + "K", + "yunki" + ], + [ + "i", + "tal" + ], + [ + "Ġà¤", + "§" + ], + [ + "Ġb", + "ounda" + ], + [ + "Ġcap", + "ital" + ], + [ + "Ġt", + "hi" + ], + [ + "ĠJ", + "ust" + ], + [ + "Ġपह", + "à¤ļ" + ], + [ + "Ġupper", + "case" + ], + [ + "2", + "7" + ], + [ + "pe", + "n" + ], + [ + "ĠW", + "o" + ], + [ + "Ġ5", + "3" + ], + [ + "Ġ50", + "5" + ], + [ + "Ġcapital", + "iz" + ], + [ + "Ġacc", + "ord" + ], + [ + "Ġav", + "g" + ], + [ + "Ġloc", + "ate" + ], + [ + "Ġaccord", + "ing" + ], + [ + "Ġin", + "ter" + ], + [ + "Ġka", + "un" + ], + [ + "Ġspeci", + "f" + ], + [ + "5", + "00" + ], + [ + "T", + "able" + ], + [ + "Ġ50", + "10" + ], + [ + "Ġfail", + "ure" + ], + [ + "Ġra", + "ha" + ], + [ + "4", + "9" + ], + [ + "K", + "itni" + ], + [ + "à¥ĭ", + "," + ], + [ + "c", + "es" + ], + [ + "f", + "ilter" + ], + [ + "k", + "ind" + ], + [ + "Ġma", + "king" + ], + [ + "Ġgo", + "ing" + ], + [ + "Ġbounda", + "ry" + ], + [ + "4", + "7" + ], + [ + "Ġt", + "reating" + ], + [ + "ĠC", + "heck" + ], + [ + "Ġd", + "uplic" + ], + [ + "Ġ5", + "2" + ], + [ + "Ġ15", + "0" + ], + [ + "3", + "3" + ], + [ + "S", + "ince" + ], + [ + "V", + "i" + ], + [ + "र", + "ह" + ], + [ + "Ġbigg", + "est" + ], + [ + "Vi", + "ew" + ], + [ + "u", + "l" + ], + [ + "ro", + "ke" + ], + [ + "Ġb", + "roke" + ], + [ + "Ġlin", + "ked" + ], + [ + "Ġcontain", + "ed" + ], + [ + "Ġdiffe", + "rent" + ], + [ + "u", + "la" + ], + [ + "Ġc", + "um" + ], + [ + "ula", + "tive" + ], + [ + "Ġcum", + "ulative" + ], + [ + "Ġw", + "or" + ], + [ + "n", + "ce" + ], + [ + "forma", + "nce" + ], + [ + "Ġ200", + "0" + ], + [ + "Ġper", + "formance" + ], + [ + "Ġ100", + "10" + ], + [ + "ढ", + "न" + ], + [ + "n", + "tion" + ], + [ + "à¤Ł", + "ल" + ], + [ + "Ġconnec", + "ted" + ], + [ + "mo", + "st" + ], + [ + "ay", + "e" + ], + [ + "tri", + "ct" + ], + [ + "Er", + "ase" + ], + [ + "Ġta", + "ra" + ], + [ + "Ġres", + "trict" + ], + [ + "E", + "mpty" + ], + [ + "h", + "on" + ], + [ + "u", + "ped" + ], + [ + "Ġ", + "gro" + ], + [ + "nt", + "hon" + ], + [ + "Ġpa", + "nthon" + ], + [ + "Ġgro", + "uped" + ], + [ + "Ġp", + "art" + ], + [ + "Ġnot", + "hing" + ], + [ + "9", + "8" + ], + [ + "Ġe", + "n" + ], + [ + "pp", + "ly" + ], + [ + "Ġj", + "is" + ], + [ + "Ġà¤ī", + "त" + ], + [ + "Ġ10", + "2" + ], + [ + "Ġla", + "g" + ], + [ + "Ġdon", + "o" + ], + [ + "5", + "3" + ], + [ + "J", + "ahan" + ], + [ + "T", + "ally" + ], + [ + "p", + "ass" + ], + [ + "Ġs", + "ur" + ], + [ + "Ġl", + "et" + ], + [ + "Ġ5", + "50" + ], + [ + "pass", + "es" + ], + [ + "Ġsur", + "passes" + ], + [ + "at", + "es" + ], + [ + "Ġp", + "ri" + ], + [ + "Ġà¤ĩस", + "ल" + ], + [ + "3", + "7" + ], + [ + "M", + "ain" + ], + [ + "Ġ7", + "0" + ], + [ + "Ġho", + "on" + ], + [ + "Ġcomplete", + "ly" + ], + [ + "Ġcap", + "ped" + ], + [ + "Ġens", + "ure" + ], + [ + "4", + "2" + ], + [ + "p", + "on" + ], + [ + "el", + "l" + ], + [ + "Ġb", + "lind" + ], + [ + "Ġस", + "ह" + ], + [ + "Ġsa", + "id" + ], + [ + "Ġसम", + "à¤Ĺ" + ], + [ + "Ġ7", + "00" + ], + [ + "Ġab", + "out" + ], + [ + "Ġdh", + "yan" + ], + [ + "o", + "te" + ], + [ + "ol", + "ds" + ], + [ + "Ġc", + "or" + ], + [ + "ĠN", + "ote" + ], + [ + "Ġà¤ī", + "पय" + ], + [ + "Ġh", + "olds" + ], + [ + "res", + "pon" + ], + [ + "Ġye", + "t" + ], + [ + "Ġcor", + "respon" + ], + [ + "Ġin", + "di" + ], + [ + "as", + "hi" + ], + [ + "Ġf", + "ashi" + ], + [ + "Ġpl", + "z" + ], + [ + "Ġfashi", + "on" + ], + [ + "7", + "2" + ], + [ + "G", + "o" + ], + [ + "T", + "ell" + ], + [ + "ha", + "i" + ], + [ + "Ġà¤ħ", + "à¤Ĺ" + ], + [ + "Ġdiffe", + "ren" + ], + [ + "Ġnikaal", + "o" + ], + [ + "Ġà¤ħव", + "र" + ], + [ + "b", + "in" + ], + [ + "Ġc", + "ho" + ], + [ + "Ġke", + "y" + ], + [ + "Ġo", + "ptions" + ], + [ + "out", + "put" + ], + [ + "Ġloc", + "ated" + ], + [ + "Ġcom", + "bin" + ], + [ + "Ġcombin", + "ed" + ], + [ + "N", + "o" + ], + [ + "i", + "fy" + ], + [ + "k", + "ari" + ], + [ + "t", + "u" + ], + [ + "Ġ", + "ti" + ], + [ + "Ġa", + "d" + ], + [ + "er", + "ify" + ], + [ + "Ġac", + "tu" + ], + [ + "Ġde", + "termine" + ], + [ + "Ġv", + "erify" + ], + [ + "Ġjaan", + "kari" + ], + [ + "Wo", + "h" + ], + [ + "Ġactu", + "ally" + ], + [ + "l", + "i" + ], + [ + "r", + "uc" + ], + [ + "t", + "ure" + ], + [ + "Ġse", + "q" + ], + [ + "Ġ5", + "20" + ], + [ + "Ġki", + "ya" + ], + [ + "à¤ļ", + "न" + ], + [ + "Ġkar", + "te" + ], + [ + "Ġcon", + "dition" + ], + [ + "Ġst", + "ruc" + ], + [ + "Ġcle", + "ar" + ], + [ + "Ġsha", + "re" + ], + [ + "Ġjis", + "me" + ], + [ + "Ġad", + "d" + ], + [ + "Ġstruc", + "ture" + ], + [ + "P", + "ur" + ], + [ + "S", + "tart" + ], + [ + "Ġ", + "gre" + ], + [ + "at", + "est" + ], + [ + "Ġread", + "ing" + ], + [ + "Ġस", + "प" + ], + [ + "Ġj", + "ya" + ], + [ + "Ġju", + "di" + ], + [ + "Pur", + "ge" + ], + [ + "Ġgre", + "atest" + ], + [ + "Ġjya", + "ada" + ], + [ + "d", + "uplic" + ], + [ + "o", + "ri" + ], + [ + "t", + "o" + ], + [ + "ad", + "ing" + ], + [ + "Ġc", + "a" + ], + [ + "à¤ķ", + "ड" + ], + [ + "Ġle", + "ading" + ], + [ + "Ġjin", + "me" + ], + [ + "Ġpri", + "or" + ], + [ + "Ġti", + "me" + ], + [ + "C", + "ompute" + ], + [ + "Ġ", + "r" + ], + [ + "न", + "न" + ], + [ + "Ġ3", + "00" + ], + [ + "Ġmo", + "de" + ], + [ + "Ġba", + "ar" + ], + [ + "Ġremo", + "ving" + ], + [ + "a", + "ar" + ], + [ + "Ġre", + "lated" + ], + [ + "us", + "aar" + ], + [ + "Ġ10", + "1" + ], + [ + "Ġan", + "usaar" + ], + [ + "Ġbad", + "e" + ], + [ + "5", + "2" + ], + [ + "5", + "8" + ], + [ + "6", + "8" + ], + [ + "a", + "k" + ], + [ + "Ġc", + "ause" + ], + [ + "Ġcol", + "s" + ], + [ + "Ġdikha", + "iye" + ], + [ + "Ġ50", + "50" + ], + [ + "ष", + "ण" + ], + [ + "Ġpe", + "ak" + ], + [ + "Ġcle", + "an" + ], + [ + "6", + "3" + ], + [ + "Ġre", + "port" + ], + [ + "ha", + "o" + ], + [ + "Ġret", + "ry" + ], + [ + "à¤Ĺ", + "र" + ], + [ + "Ġretry", + "ing" + ], + [ + "7", + "4" + ], + [ + "Y", + "ou" + ], + [ + "k", + "a" + ], + [ + "l", + "ook" + ], + [ + "Ġ", + "ri" + ], + [ + "à¤", + "Ĩ" + ], + [ + "Ġp", + "ich" + ], + [ + "़", + "à¥ĩà¤Ĥ" + ], + [ + "Ġर", + "à¤ĸन" + ], + [ + "gh", + "t" + ], + [ + "Ġri", + "ght" + ], + [ + "6", + "2" + ], + [ + "य", + "र" + ], + [ + "Ġà¤ĩस", + "न" + ], + [ + "A", + "gar" + ], + [ + "Ġkind", + "ly" + ], + [ + "ĠS", + "how" + ], + [ + "Ġd", + "obara" + ], + [ + "Ġà¤ĩस", + "à¤ķ" + ], + [ + "Ġhu", + "a" + ], + [ + "I", + "de" + ], + [ + "T", + "o" + ], + [ + "n", + "tify" + ], + [ + "Ġw", + "ipe" + ], + [ + "an", + "e" + ], + [ + "Ġ3", + "1" + ], + [ + "व", + "श" + ], + [ + "Ġà¤Ĩ", + "à¤Ī" + ], + [ + "Ġà¤Ĩ", + "वश" + ], + [ + "Ġzaro", + "or" + ], + [ + "Ide", + "ntify" + ], + [ + "D", + "el" + ], + [ + "Ġm", + "il" + ], + [ + "Ġ11", + "00" + ], + [ + "Del", + "ete" + ], + [ + "d", + "l" + ], + [ + "v", + "al" + ], + [ + "he", + "n" + ], + [ + "Ġre", + "gar" + ], + [ + "à¥Ģ", + "à¤Ĥ।" + ], + [ + "Ġ2", + "10" + ], + [ + "Ġinter", + "val" + ], + [ + "Ġspecif", + "ically" + ], + [ + "dl", + "ess" + ], + [ + "Ġregar", + "dless" + ], + [ + "e", + "ed" + ], + [ + "s", + "s" + ], + [ + "ro", + "ss" + ], + [ + "Ġac", + "ross" + ], + [ + "l", + "iye" + ], + [ + "ĠT", + "hen" + ], + [ + "Ġis", + "liye" + ], + [ + "Ġ2", + "50" + ], + [ + "Ġna", + "am" + ], + [ + "à¤ħ", + "à¤Ĺर" + ], + [ + "Ġ5", + "10" + ], + [ + "N", + "eed" + ], + [ + "Ġ5", + "1" + ], + [ + "Ġ10", + "50" + ], + [ + "b", + "hai" + ], + [ + "ou", + "gh" + ], + [ + "Ġhave", + "nt" + ], + [ + "A", + "a" + ], + [ + "Ġs", + "irf" + ], + [ + "Ġun", + "ka" + ], + [ + "kh", + "ri" + ], + [ + "Aa", + "khri" + ], + [ + "ing", + "le" + ], + [ + "ĠI", + "sk" + ], + [ + "Ġs", + "ingle" + ], + [ + "Ġdhund", + "ho" + ], + [ + "w", + "ar" + ], + [ + "ĠA", + "nswer" + ], + [ + "Ġon", + "war" + ], + [ + "ĠB", + "atao" + ], + [ + "Ġ5", + "100" + ], + [ + "ĠIsk", + "a" + ], + [ + "Ġonwar", + "ds" + ], + [ + "b", + "atao" + ], + [ + "Ġ10", + "500" + ], + [ + "ata", + "iye" + ], + [ + "Ġcapitaliz", + "ation" + ], + [ + "Ġjisme", + "in" + ], + [ + "7", + "7" + ], + [ + "Ġ1", + "20" + ], + [ + "à¥ĭà¤Ĥ", + "।" + ], + [ + "u", + "is" + ], + [ + "ting", + "uis" + ], + [ + "Ġà¤ħ", + "त" + ], + [ + "Ġdis", + "tinguis" + ], + [ + "Ġdistinguis", + "hing" + ], + [ + "IJ", + "स" + ], + [ + "Ġ50", + "1" + ], + [ + "Ġ9", + "2" + ], + [ + "Ġà¤", + "IJस" + ], + [ + "म", + "à¤ļ" + ], + [ + "Ġ10", + "100" + ], + [ + "3", + "2" + ], + [ + "3", + "5" + ], + [ + "6", + "5" + ], + [ + "7", + "9" + ], + [ + "ĠC", + "ount" + ], + [ + "à¤ī", + "स" + ], + [ + "Ġja", + "ata" + ], + [ + "Ġtara", + "h" + ], + [ + "6", + "9" + ], + [ + "9", + "4" + ], + [ + "ti", + "re" + ], + [ + "or", + "a" + ], + [ + "ic", + "e" + ], + [ + "Ġp", + "ao" + ], + [ + "à¤Ĺ", + "त" + ], + [ + "Ġpo", + "ora" + ], + [ + "Ġen", + "tire" + ], + [ + "5", + "9" + ], + [ + "S", + "um" + ], + [ + "S", + "irf" + ], + [ + "ye", + "in" + ], + [ + "Ġwa", + "hi" + ], + [ + "Ġdes", + "c" + ], + [ + "Ġbata", + "yein" + ], + [ + "Ġdekh", + "na" + ], + [ + "Ġres", + "ul" + ], + [ + "Ġpa", + "as" + ], + [ + "Ġbel", + "ong" + ], + [ + "k", + "ya" + ], + [ + "ĵ", + "र" + ], + [ + "se", + "qu" + ], + [ + "Ġà¤", + "ĵर" + ], + [ + "Ġs", + "ub" + ], + [ + "Ġk", + "h" + ], + [ + "Ġ1", + "11" + ], + [ + "Ġme", + "ntion" + ], + [ + "Ġà¤ī", + "ल" + ], + [ + "Ġli", + "es" + ], + [ + "Ġत", + "रह" + ], + [ + "Ġlet", + "ter" + ], + [ + "Ġdifferen", + "ces" + ], + [ + "sequ", + "ent" + ], + [ + "Ġsub", + "sequent" + ], + [ + "3", + "1" + ], + [ + "8", + "8" + ], + [ + "C", + "le" + ], + [ + "h", + "ta" + ], + [ + "ij", + "र" + ], + [ + "alan", + "ki" + ], + [ + "Ġka", + "re" + ], + [ + "ड", + "र" + ], + [ + "Ġh", + "alanki" + ], + [ + "Ġcha", + "hta" + ], + [ + "Ġjaha", + "an" + ], + [ + "Ġduplic", + "ates" + ], + [ + "Ġwor", + "k" + ], + [ + "Ġresul", + "ts" + ], + [ + "Cle", + "ar" + ], + [ + "al", + "i" + ], + [ + "tic", + "al" + ], + [ + "Ġlist", + "ed" + ], + [ + "Ġ4", + "5" + ], + [ + "Ġiden", + "tical" + ], + [ + "Ġ14", + "3" + ], + [ + "J", + "in" + ], + [ + "b", + "lind" + ], + [ + "p", + "ari" + ], + [ + "Ġc", + "reating" + ], + [ + "da", + "r" + ], + [ + "Ġpad", + "hi" + ], + [ + "Ġan", + "dar" + ], + [ + "Ġbad", + "i" + ], + [ + "़à¥ĩ", + "।" + ], + [ + "Ġjaan", + "i" + ], + [ + "Ġcom", + "pari" + ], + [ + "Ġkh", + "ali" + ], + [ + "Ġcompari", + "son" + ], + [ + "A", + "c" + ], + [ + "B", + "ataiye" + ], + [ + "a", + "ki" + ], + [ + "se", + "lect" + ], + [ + "Ġa", + "ao" + ], + [ + "pec", + "ting" + ], + [ + "Ġse", + "nsitive" + ], + [ + "स", + "à¤ķ" + ], + [ + "ĠS", + "tart" + ], + [ + "Ġ50", + "2" + ], + [ + "Ġ15", + "10" + ], + [ + "Ġnumber", + "s" + ], + [ + "Ġjink", + "o" + ], + [ + "Ġ6", + "2" + ], + [ + "Ġlin", + "king" + ], + [ + "Ġins", + "pecting" + ], + [ + "Ġchho", + "ta" + ], + [ + "Ġduplic", + "ate" + ], + [ + "4", + "3" + ], + [ + "L", + "oc" + ], + [ + "O", + "pen" + ], + [ + "o", + "ries" + ], + [ + "u", + "al" + ], + [ + "u", + "ring" + ], + [ + "he", + "ad" + ], + [ + "Ġa", + "head" + ], + [ + "Ġc", + "ateg" + ], + [ + "pu", + "lated" + ], + [ + "vid", + "ual" + ], + [ + "म", + "à¤ķ" + ], + [ + "Ġbe", + "en" + ], + [ + "Ġg", + "ot" + ], + [ + "Ġgay", + "i" + ], + [ + "Ġpo", + "pulated" + ], + [ + "Ġens", + "uring" + ], + [ + "Ġindi", + "vidual" + ], + [ + "Ġkare", + "in" + ], + [ + "Loc", + "ate" + ], + [ + "Ġcateg", + "ories" + ], + [ + "'", + "d" + ], + [ + "l", + "o" + ], + [ + "u", + "gh" + ], + [ + "ti", + "yan" + ], + [ + "ro", + "ugh" + ], + [ + "mo", + "ve" + ], + [ + "Re", + "move" + ], + [ + "Ġp", + "an" + ], + [ + "Ġf", + "lo" + ], + [ + "ch", + "tiyan" + ], + [ + "Ġप", + "ड" + ], + [ + "म", + "त" + ], + [ + "Ġth", + "rough" + ], + [ + "Ġdikha", + "na" + ], + [ + "à¤ľ", + "न" + ], + [ + "Ġ50", + "3" + ], + [ + "à¤ħ", + "ब" + ], + [ + "Ġ9", + "1" + ], + [ + "Ġta", + "aki" + ], + [ + "Ġagain", + "st" + ], + [ + "Ġdikh", + "aye" + ], + [ + "Ġbigg", + "er" + ], + [ + "Ġpan", + "chtiyan" + ], + [ + "Ġflo", + "or" + ], + [ + "2", + "9" + ], + [ + "g", + "an" + ], + [ + "n", + "ing" + ], + [ + "o", + "d" + ], + [ + "Ġin", + "sensitive" + ], + [ + "à¥ĩ", + "," + ], + [ + "Ġb", + "ring" + ], + [ + "Ġk", + "ho" + ], + [ + "it", + "ne" + ], + [ + "न", + "श" + ], + [ + "य", + "म" + ], + [ + "़", + "à¥ĩà¤Ĥ," + ], + [ + "द", + "नश" + ], + [ + "व", + "à¤ľà¤¨" + ], + [ + "Ġor", + "gan" + ], + [ + "Ġpl", + "us" + ], + [ + "Ġex", + "am" + ], + [ + "Ġsor", + "ting" + ], + [ + "Ġdoes", + "n" + ], + [ + "Ġal", + "one" + ], + [ + "iz", + "ed" + ], + [ + "Ġrun", + "ning" + ], + [ + "Ġcapitaliz", + "ed" + ], + [ + "Ġlag", + "ao" + ], + [ + "Ġdikhaye", + "in" + ], + [ + "Ġkho", + "j" + ], + [ + "Ġorgan", + "ized" + ], + [ + "Ġexam", + "ine" + ], + [ + "4", + "0" + ], + [ + "7", + "5" + ], + [ + "A", + "ap" + ], + [ + "K", + "itne" + ], + [ + "S", + "ort" + ], + [ + "a", + "na" + ], + [ + "b", + "er" + ], + [ + "b", + "si" + ], + [ + "k", + "ul" + ], + [ + "t", + "hao" + ], + [ + "½", + "Ð" + ], + [ + "Ð", + "½Ð" + ], + [ + "Ġa", + "pply" + ], + [ + "ा", + "-" + ], + [ + "म", + "нÐ" + ], + [ + "ह", + "र" + ], + [ + "il", + "kul" + ], + [ + "Ġu", + "thao" + ], + [ + "Ġconnec", + "ting" + ], + [ + "ĠK", + "itni" + ], + [ + "Ġye", + "h" + ], + [ + "à¤ħ", + "स" + ], + [ + "Ġlin", + "ks" + ], + [ + "Ġwe", + "bsi" + ], + [ + "Ġra", + "he" + ], + [ + "Ġcorrespon", + "ding" + ], + [ + "Ġcho", + "te" + ], + [ + "मнÐ", + "µ" + ], + [ + "Ġwebsi", + "te" + ], + [ + "A", + "pply" + ], + [ + "W", + "ipe" + ], + [ + "e", + "p" + ], + [ + "i", + "od" + ], + [ + "m", + "ber" + ], + [ + "n", + "umber" + ], + [ + "me", + "mber" + ], + [ + "Ġre", + "member" + ], + [ + "Ġw", + "an" + ], + [ + "Ġm", + "ind" + ], + [ + "Ġb", + "ilkul" + ], + [ + "Ġp", + "rint" + ], + [ + "Ġke", + "ep" + ], + [ + "Ġश", + "हर" + ], + [ + "Ġper", + "iod" + ], + [ + "Ġer", + "ase" + ], + [ + "Ġdele", + "ting" + ], + [ + "duplic", + "ate" + ], + [ + "Ġwan", + "na" + ], + [ + "Ġkeep", + "ing" + ], + [ + "5", + "4" + ], + [ + "K", + "ind" + ], + [ + "c", + "ess" + ], + [ + "p", + "pens" + ], + [ + "r", + "ab" + ], + [ + "Ġ", + "unit" + ], + [ + "Ġac", + "che" + ], + [ + "य", + "à¤ķत" + ], + [ + "Ġha", + "ta" + ], + [ + "Ġha", + "ppens" + ], + [ + "à¥Ĥ", + "à¤ģ" + ], + [ + "Ġà¤ħ", + "à¤Ń" + ], + [ + "Ġtotal", + "ly" + ], + [ + "usive", + "ly" + ], + [ + "Ġfilte", + "ring" + ], + [ + "Ġdate", + "d" + ], + [ + "Ġexcl", + "usively" + ], + [ + "Ġzaro", + "ori" + ], + [ + "Ġlag", + "ana" + ], + [ + "Ġca", + "ring" + ], + [ + "Ac", + "cess" + ], + [ + "A", + "s" + ], + [ + "V", + "ac" + ], + [ + "d", + "y" + ], + [ + "i", + "er" + ], + [ + "i", + "or" + ], + [ + "m", + "ier" + ], + [ + "u", + "per" + ], + [ + "Ġs", + "uper" + ], + [ + "Ġin", + "to" + ], + [ + "ा", + "," + ], + [ + "an", + "i" + ], + [ + "र", + "à¤ļन" + ], + [ + "Ġj", + "ane" + ], + [ + "Ġpre", + "mier" + ], + [ + "Ġcha", + "he" + ], + [ + "Ġba", + "are" + ], + [ + "Ġstu", + "dy" + ], + [ + "Ġpa", + "aye" + ], + [ + "Ġdikh", + "ate" + ], + [ + "Ġpadh", + "ni" + ], + [ + "Kind", + "ly" + ], + [ + "Vac", + "ate" + ], + [ + "Ġsuper", + "ior" + ], + [ + "p", + "oints" + ], + [ + "s", + "fy" + ], + [ + "Ġ", + "+" + ], + [ + "ti", + "sfy" + ], + [ + "ay", + "en" + ], + [ + "Ġlabel", + "s" + ], + [ + "Ġto", + "o" + ], + [ + "Ġneed", + "s" + ], + [ + "Ġsa", + "tisfy" + ], + [ + "Ġà¤ļ", + "र" + ], + [ + "Un", + "hi" + ], + [ + "im", + "al" + ], + [ + "Ġmin", + "imal" + ], + [ + "Ġमह", + "त" + ], + [ + "Ġ14", + "1" + ], + [ + "Ġend", + "points" + ], + [ + "Ġdikh", + "ayen" + ], + [ + "Ġcondition", + "s" + ], + [ + "Ġbelong", + "s" + ], + [ + "Ġdikhayen", + "ge" + ], + [ + "6", + "4" + ], + [ + "7", + "3" + ], + [ + "J", + "oin" + ], + [ + "b", + "le" + ], + [ + "g", + "a" + ], + [ + "s", + "ue" + ], + [ + "Ġ", + "pur" + ], + [ + "ma", + "ntion" + ], + [ + "Ġà¤", + "ijर" + ], + [ + "ay", + "i" + ], + [ + "Ġis", + "sue" + ], + [ + "pl", + "s" + ], + [ + "Ġd", + "ocu" + ], + [ + "य", + "व" + ], + [ + "द", + "र" + ], + [ + "Ġdo", + "ing" + ], + [ + "Ġla", + "ayi" + ], + [ + "Ġavail", + "ble" + ], + [ + "what", + "s" + ], + [ + "à¤ħ", + "रà¤ķ" + ], + [ + "Ġfi", + "x" + ], + [ + "Ġremo", + "ve" + ], + [ + "Ġcause", + "d" + ], + [ + "Ġmil", + "ao" + ], + [ + "Ġpur", + "ge" + ], + [ + "Ġdocu", + "mantion" + ], + [ + "यव", + "स" + ], + [ + "S", + "tri" + ], + [ + "Ġ", + "term" + ], + [ + "nt", + "ory" + ], + [ + "Ġs", + "hi" + ], + [ + "Ġs", + "iz" + ], + [ + "Ġin", + "ve" + ], + [ + "Ġl", + "ine" + ], + [ + "ĠS", + "o" + ], + [ + "Ġ3", + "9" + ], + [ + "Ġthat", + "s" + ], + [ + "Ġत", + "ल" + ], + [ + "Ġ12", + "8" + ], + [ + "Ġexcl", + "uding" + ], + [ + "Ġpadh", + "na" + ], + [ + "Ġcorrespon", + "ds" + ], + [ + "Stri", + "p" + ], + [ + "Ġterm", + "s" + ], + [ + "Ġshi", + "row" + ], + [ + "Ġsiz", + "e" + ], + [ + "Ġinve", + "ntory" + ], + [ + "W", + "e" + ], + [ + "]", + "." + ], + [ + "g", + "o" + ], + [ + "n", + "ces" + ], + [ + "u", + "k" + ], + [ + "Ġ", + "qui" + ], + [ + "Ġin", + "sta" + ], + [ + "mploye", + "e" + ], + [ + "Ġki", + "ye" + ], + [ + "Ġà¤ļ", + "ढ" + ], + [ + "Ġas", + "ap" + ], + [ + "à¤Ľ", + "त" + ], + [ + "Pich", + "li" + ], + [ + "Ġwor", + "ry" + ], + [ + "Ġr", + "uk" + ], + [ + "Ġqui", + "ck" + ], + [ + "Ġinsta", + "nces" + ], + [ + "2", + "5" + ], + [ + "3", + "8" + ], + [ + "8", + "0" + ], + [ + "8", + "4" + ], + [ + "9", + "1" + ], + [ + "R", + "un" + ], + [ + "h", + "na" + ], + [ + "s", + "irf" + ], + [ + "u", + "red" + ], + [ + "Ġ", + "ran" + ], + [ + "as", + "ured" + ], + [ + "Ġe", + "mployee" + ], + [ + "Ġon", + "ce" + ], + [ + "Ġde", + "in" + ], + [ + "Ġme", + "asured" + ], + [ + "प", + "म" + ], + [ + "à¥Ĥ", + "à¤Ĥ।" + ], + [ + "ik", + "r" + ], + [ + "Ġत", + "थ" + ], + [ + "Ġz", + "ikr" + ], + [ + "Ġ7", + "3" + ], + [ + "Ġno", + "ting" + ], + [ + "Ġag", + "le" + ], + [ + "Ġdhund", + "hna" + ], + [ + "Ġarrang", + "ing" + ], + [ + "Ġdikh", + "ani" + ], + [ + "Ġpich", + "li" + ], + [ + "Ġran", + "k" + ], + [ + "'", + "." + ], + [ + "6", + "1" + ], + [ + "t", + "her" + ], + [ + "Ġ", + "Re" + ], + [ + "Ġwhe", + "ther" + ], + [ + "vo", + "ice" + ], + [ + "stude", + "nt" + ], + [ + "Ġma", + "de" + ], + [ + "à¤ļ", + "य" + ], + [ + "Ġhi", + "t" + ], + [ + "vi", + "ew" + ], + [ + "Ġpad", + "ta" + ], + [ + "Ġjaan", + "ch" + ], + [ + "Ġexcl", + "ude" + ], + [ + "Ġtara", + "f" + ], + [ + "Ġr", + "id" + ], + [ + "ĠRe", + "view" + ], + [ + "3", + "9" + ], + [ + "c", + "tion" + ], + [ + "c", + "ates" + ], + [ + "i", + "se" + ], + [ + "n", + "ull" + ], + [ + "p", + "res" + ], + [ + "s", + "um" + ], + [ + "Ġ", + "student" + ], + [ + "on", + "d" + ], + [ + "in", + "a" + ], + [ + "in", + "k" + ], + [ + "se", + "d" + ], + [ + "se", + "ction" + ], + [ + "te", + "rs" + ], + [ + "Ġb", + "ina" + ], + [ + "ĠL", + "ook" + ], + [ + "Ġth", + "ink" + ], + [ + "ĠG", + "o" + ], + [ + "ड", + "à¤ķ" + ], + [ + "Ġà¤ħ", + "ल" + ], + [ + "Ġg", + "rab" + ], + [ + "Ġex", + "pres" + ], + [ + "Ġ9", + "3" + ], + [ + "Ġcap", + "s" + ], + [ + "Ġinter", + "section" + ], + [ + "Ġspecif", + "ic" + ], + [ + "Ġlet", + "ters" + ], + [ + "Ġindi", + "cates" + ], + [ + "Ġpich", + "la" + ], + [ + "Ġexpres", + "sed" + ], + [ + "e", + "ts" + ], + [ + "f", + "in" + ], + [ + "i", + "tion" + ], + [ + "in", + "est" + ], + [ + "Ġa", + "vo" + ], + [ + "Ġde", + "fin" + ], + [ + "Ġf", + "inest" + ], + [ + "Ġd", + "ho" + ], + [ + "Ġme", + "ets" + ], + [ + "म", + "हत" + ], + [ + "Ġव", + "à¤ľà¤¹" + ], + [ + "à¥Ĥ", + "à¤ģ," + ], + [ + "Ġkar", + "an" + ], + [ + "Ġsmall", + "er" + ], + [ + "Ġho", + "ta" + ], + [ + "Ġpo", + "ori" + ], + [ + "Ġà¤Ĩà¤ķ", + "ल" + ], + [ + "Ġen", + "tity" + ], + [ + "Ġcho", + "ose" + ], + [ + "to", + "tal" + ], + [ + "Ġca", + "me" + ], + [ + "To", + "p" + ], + [ + "ition", + "s" + ], + [ + "Ġavo", + "id" + ], + [ + "Ġdefin", + "itions" + ], + [ + "Ġdho", + "ond" + ], + [ + "2", + "4" + ], + [ + "9", + "5" + ], + [ + "w", + "hich" + ], + [ + "Ġde", + "p" + ], + [ + "Ġप", + "ur" + ], + [ + "म", + "à¤Ĺ" + ], + [ + "Ġka", + "ise" + ], + [ + "Ġ5", + "9" + ], + [ + "Ġ10", + "4" + ], + [ + "Ġmin", + "us" + ], + [ + "Ġ6", + "3" + ], + [ + "Ġ6", + "6" + ], + [ + "Ġ4", + "4" + ], + [ + "Ġ8", + "9" + ], + [ + "Ġknow", + "n" + ], + [ + "duplic", + "ated" + ], + [ + "Ġzaroor", + "at" + ], + [ + "Ġपur", + "ga" + ], + [ + "c", + "alculate" + ], + [ + "ar", + "tments" + ], + [ + "Ġin", + "voice" + ], + [ + "र", + "à¤ħ" + ], + [ + "य", + "à¤ķ" + ], + [ + "Ġdisplay", + "ing" + ], + [ + "Ġth", + "ough" + ], + [ + "Ġन", + "à¤ľ" + ], + [ + "Ġo", + "pen" + ], + [ + "11", + "1" + ], + [ + "Ġnikal", + "e" + ], + [ + "Ġconn", + "ect" + ], + [ + "Ġà¤ĸ", + "र" + ], + [ + "Ġab", + "hi" + ], + [ + "Ġbata", + "iye" + ], + [ + "Ġsame", + "t" + ], + [ + "Ġdep", + "artments" + ], + [ + "Ġà¤¨à¤ľ", + "रà¤ħ" + ], + [ + "3", + "66" + ], + [ + "4", + "6" + ], + [ + "A", + "b" + ], + [ + "a", + "ke" + ], + [ + "n", + "er" + ], + [ + "u", + "des" + ], + [ + "w", + "are" + ], + [ + "Ġ", + "string" + ], + [ + "Ġa", + "ware" + ], + [ + "Ġb", + "ound" + ], + [ + "Ġ1", + "19" + ], + [ + "Ġ3", + "5" + ], + [ + "Ġincl", + "udes" + ], + [ + "08", + "5" + ], + [ + "Ġ13", + "9" + ], + [ + "Ġch", + "al" + ], + [ + "Ġ14", + "8" + ], + [ + "Ġà¤ħà¤Ĺ", + "ल" + ], + [ + "Ġquick", + "ly" + ], + [ + "Ġbound", + "s" + ], + [ + "6", + "7" + ], + [ + "a", + "ao" + ], + [ + "b", + "as" + ], + [ + "c", + "ompute" + ], + [ + "ti", + "er" + ], + [ + "ubscri", + "ption" + ], + [ + "it", + "ting" + ], + [ + "app", + "ly" + ], + [ + "à¤Ł", + "म" + ], + [ + "à¤ħ", + "à¤ľ" + ], + [ + "Ġà¤ħव", + "श" + ], + [ + "Ġchal", + "aao" + ], + [ + "2", + "8" + ], + [ + "8", + "6" + ], + [ + "c", + "o" + ], + [ + "i", + "ed" + ], + [ + "m", + "itting" + ], + [ + "n", + "ner" + ], + [ + "er", + "ating" + ], + [ + "Ġs", + "ubscription" + ], + [ + "pl", + "ied" + ], + [ + "ub", + "mitting" + ], + [ + "Ġस", + "ब" + ], + [ + "Ġ2", + "1" + ], + [ + "en", + "erating" + ], + [ + "Ġsup", + "plied" + ], + [ + "Ġma", + "nner" + ], + [ + "Ġ10", + "7" + ], + [ + "Ġg", + "enerating" + ], + [ + "th", + "ough" + ], + [ + "Ġdis", + "co" + ], + [ + "Ġ9", + "7" + ], + [ + "Ġ4", + "2" + ], + [ + "Ġal", + "though" + ], + [ + "Ġ13", + "2" + ], + [ + "Ġres", + "ubmitting" + ], + [ + "Ġ14", + "9" + ], + [ + "Ġdifferen", + "ce" + ], + [ + "Ġdisco", + "ver" + ], + [ + "9", + "7" + ], + [ + "a", + "fter" + ], + [ + "i", + "ties" + ], + [ + "l", + "es" + ], + [ + "Ġ", + "tit" + ], + [ + "Ġc", + "ities" + ], + [ + "Ġt", + "um" + ], + [ + "ay", + "ment" + ], + [ + "Ġs", + "ari" + ], + [ + "Ġpro", + "duc" + ], + [ + "Ġtit", + "les" + ], + [ + "Ġproduc", + "t" + ], + [ + "4", + "8" + ], + [ + "T", + "ake" + ], + [ + "w", + "o" + ], + [ + "Ġ1", + "22" + ], + [ + "Ġ3", + "0" + ], + [ + "Ġ3", + "7" + ], + [ + "12", + "4" + ], + [ + "Ġ4", + "0" + ], + [ + "Ġà¤ĩ", + "न" + ], + [ + "Ġ8", + "0" + ], + [ + "Ġ8", + "4" + ], + [ + "Ġ13", + "0" + ], + [ + "'", + "?" + ], + [ + "3", + "6" + ], + [ + "a", + "ve" + ], + [ + "Ġ", + "ir" + ], + [ + "य", + "न" + ], + [ + "य", + "यन" + ], + [ + "Ġg", + "ave" + ], + [ + "res", + "pective" + ], + [ + "हत", + "र" + ], + [ + "Ġsha", + "red" + ], + [ + "Ġmil", + "ane" + ], + [ + "Ġir", + "respective" + ], + [ + "Ġ1", + "9" + ], + [ + "Ġ2", + "2" + ], + [ + "Ġ10", + "10" + ], + [ + "Ġ9", + "8" + ], + [ + "Ġ55", + "00" + ], + [ + "2", + "00" + ], + [ + "7", + "0" + ], + [ + "W", + "ith" + ], + [ + "k", + "hao" + ], + [ + "Ġm", + "ust" + ], + [ + "à¤ĸ", + "र" + ], + [ + "With", + "out" + ], + [ + "4", + "5" + ], + [ + "9", + "9" + ], + [ + "r", + "se" + ], + [ + "Ġc", + "ou" + ], + [ + "Ġis", + "n" + ], + [ + "hip", + "ment" + ], + [ + "à¤ķ", + "ल" + ], + [ + "Ġkind", + "s" + ], + [ + "Ġ1", + "7" + ], + [ + "Ġ1", + "13" + ], + [ + "Ġ5", + "8" + ], + [ + "Ġ10", + "5" + ], + [ + "Ġ10", + "6" + ], + [ + "Ġ3", + "8" + ], + [ + "14", + "3" + ], + [ + "14", + "8" + ], + [ + "Ġ4", + "7" + ], + [ + "Ġ4", + "8" + ], + [ + "Ġ13", + "6" + ], + [ + "pen", + "se" + ], + [ + "Ġcou", + "rse" + ], + [ + "I", + "t" + ], + [ + "15", + "00" + ], + [ + "10", + "7" + ], + [ + "Ġex", + "pense" + ], + [ + "1", + "20" + ], + [ + "3", + "23" + ], + [ + "6", + "03" + ], + [ + "8", + "00" + ], + [ + "Ġs", + "hipment" + ], + [ + "Ġ10", + "9" + ], + [ + "13", + "0" + ], + [ + "13", + "9" + ], + [ + "Ġ6", + "1" + ], + [ + "Ġ12", + "9" + ], + [ + "Ġ13", + "1" + ], + [ + "Ġà¤ĩन", + "व" + ], + [ + "Ġp", + "ayment" + ], + [ + "Ġregion", + "s" + ], + [ + "Ġun", + "hi" + ], + [ + "13", + "5" + ], + [ + "9", + "0" + ], + [ + "9", + "96" + ], + [ + "Ġ1", + "18" + ], + [ + "Ġ1", + "17" + ], + [ + "Ġ5", + "4" + ], + [ + "10", + "1" + ], + [ + "11", + "3" + ], + [ + "Ġ7", + "9" + ], + [ + "Ġ9", + "9" + ], + [ + "Ġ12", + "4" + ], + [ + "07", + "5" + ], + [ + "56", + "4" + ], + [ + "Ġlabel", + "ed" + ], + [ + "ह", + "à¤ķ" + ], + [ + "14", + "1" + ], + [ + "1", + "15" + ], + [ + "H", + "e" + ], + [ + "Ġacc", + "ount" + ], + [ + "Ġ3", + "6" + ], + [ + "10", + "6" + ], + [ + "12", + "8" + ], + [ + "Ġ13", + "8" + ], + [ + "He", + "y" + ], + [ + "B", + "hai" + ], + [ + "Ġw", + "ho" + ], + [ + "Ġthe", + "y" + ], + [ + "13", + "8" + ], + [ + "100", + "0" + ], + [ + "100", + "00" + ], + [ + "5", + "5" + ], + [ + "8", + "2" + ], + [ + "8", + "5" + ], + [ + "C", + "ourses" + ], + [ + "Ġrating", + "s" + ], + [ + "Ġ2", + "7" + ], + [ + "Ġà¤ħ", + "प" + ], + [ + "12", + "7" + ], + [ + "Ġ6", + "8" + ], + [ + "Ġ12", + "7" + ], + [ + "Ġ8", + "1" + ], + [ + "v", + "ice" + ], + [ + "Ġde", + "vice" + ], + [ + "स", + "à¤Ń" + ], + [ + "10", + "4" + ], + [ + "202", + "1" + ], + [ + "Ġà¤Ĩà¤Ī", + "ड" + ], + [ + "2", + "96" + ], + [ + "3", + "87" + ], + [ + "3", + "47" + ], + [ + "5", + "18" + ], + [ + "7", + "56" + ], + [ + "7", + "89" + ], + [ + "8", + "1" + ], + [ + "8", + "03" + ], + [ + "8", + "44" + ], + [ + "8", + "49" + ], + [ + "8", + "366" + ], + [ + "9", + "66" + ], + [ + "or", + "es" + ], + [ + "Ġ1", + "21" + ], + [ + "Ġ2", + "9" + ], + [ + "Ġ5", + "6" + ], + [ + "10", + "3" + ], + [ + "11", + "4" + ], + [ + "11", + "9" + ], + [ + "Ġ15", + "50" + ], + [ + "18", + "09" + ], + [ + "Ġ6", + "7" + ], + [ + "Ġ7", + "1" + ], + [ + "Ġ4", + "3" + ], + [ + "Ġ13", + "3" + ], + [ + "Ġ13", + "4" + ], + [ + "Ġ13", + "5" + ], + [ + "66", + "3" + ], + [ + "93", + "4" + ], + [ + "87", + "57" + ], + [ + "27", + "08" + ], + [ + "46", + "52" + ], + [ + "323", + "3" + ], + [ + "i", + "ke" + ], + [ + "t", + "ment" + ], + [ + "Ġ", + "vis" + ], + [ + "Ġa", + "ppoin" + ], + [ + "di", + "khao" + ], + [ + "Ġb", + "ook" + ], + [ + "ĠL", + "ike" + ], + [ + "13", + "2" + ], + [ + "14", + "9" + ], + [ + "Ġvis", + "it" + ], + [ + "Ġappoin", + "tment" + ], + [ + "3", + "564" + ], + [ + "4", + "14" + ], + [ + "4", + "53" + ], + [ + "6", + "00" + ], + [ + "6", + "06" + ], + [ + "7", + "22" + ], + [ + "9", + "44" + ], + [ + "9", + "72" + ], + [ + "a", + "are" + ], + [ + "ab", + "hi" + ], + [ + "Ġs", + "c" + ], + [ + "Ġ1", + "14" + ], + [ + "Ġdescription", + "s" + ], + [ + "Ġ2", + "5" + ], + [ + "Ġ2", + "8" + ], + [ + "Ġ5", + "7" + ], + [ + "Ġh", + "on" + ], + [ + "Ġ3", + "2" + ], + [ + "Ġ3", + "3" + ], + [ + "ष", + "à¤ķ" + ], + [ + "12", + "2" + ], + [ + "14", + "2" + ], + [ + "Ġश", + "ब" + ], + [ + "Ġ6", + "5" + ], + [ + "Ġ6", + "9" + ], + [ + "Ġ7", + "4" + ], + [ + "Ġ7", + "5" + ], + [ + "Ġ7", + "6" + ], + [ + "Ġ7", + "7" + ], + [ + "Ġ9", + "0" + ], + [ + "Ġ9", + "6" + ], + [ + "Ġ4", + "9" + ], + [ + "Ġ12", + "6" + ], + [ + "Ġ8", + "7" + ], + [ + "Ġ13", + "7" + ], + [ + "07", + "6" + ], + [ + "Ġ14", + "2" + ], + [ + "Ġ14", + "4" + ], + [ + "Ġ14", + "5" + ], + [ + "Ġ14", + "7" + ], + [ + "92", + "4" + ], + [ + "89", + "9" + ], + [ + "42", + "52" + ], + [ + "94", + "02" + ], + [ + "94", + "82" + ], + [ + "k", + "ha" + ], + [ + "l", + "ed" + ], + [ + "Ġk", + "uch" + ], + [ + "स", + "ब" + ], + [ + "Ġli", + "kha" + ], + [ + "500", + "0" + ], + [ + "Ġsc", + "ores" + ], + [ + "4", + "1" + ], + [ + "13", + "6" + ], + [ + "Ġfil", + "led" + ], + [ + "b", + "ataiye" + ], + [ + "त", + "à¤ķ" + ], + [ + "ल", + "र" + ], + [ + "10", + "9" + ], + [ + "11", + "6" + ], + [ + "12", + "9" + ], + [ + "13", + "1" + ], + [ + "3", + "4" + ], + [ + "5", + "1" + ], + [ + "8", + "3" + ], + [ + "O", + "r" + ], + [ + "T", + "ic" + ], + [ + "d", + "er" + ], + [ + "à¤", + "ijर" + ], + [ + "Ġde", + "khao" + ], + [ + "10", + "2" + ], + [ + "11", + "7" + ], + [ + "11", + "8" + ], + [ + "12", + "5" + ], + [ + "à¤ĩà¤Ł", + "ल" + ], + [ + "Or", + "der" + ], + [ + "Tic", + "ke" + ], + [ + "E", + "mployees" + ], + [ + "tab", + "on" + ], + [ + "Ġà¤Ĺ", + "à¤ı" + ], + [ + "2", + "19" + ], + [ + "2", + "49" + ], + [ + "2", + "47" + ], + [ + "2", + "68" + ], + [ + "3", + "08" + ], + [ + "4", + "08" + ], + [ + "4", + "56" + ], + [ + "4", + "96" + ], + [ + "4", + "37" + ], + [ + "4", + "63" + ], + [ + "4", + "138" + ], + [ + "5", + "02" + ], + [ + "5", + "33" + ], + [ + "5", + "124" + ], + [ + "5", + "107" + ], + [ + "5", + "101" + ], + [ + "6", + "87" + ], + [ + "6", + "27" + ], + [ + "6", + "47" + ], + [ + "6", + "74" + ], + [ + "6", + "085" + ], + [ + "6", + "124" + ], + [ + "6", + "130" + ], + [ + "7", + "09" + ], + [ + "7", + "58" + ], + [ + "7", + "62" + ], + [ + "7", + "88" + ], + [ + "7", + "120" + ], + [ + "8", + "16" + ], + [ + "8", + "23" + ], + [ + "8", + "113" + ], + [ + "8", + "114" + ], + [ + "9", + "17" + ], + [ + "9", + "49" + ], + [ + "9", + "53" + ], + [ + "9", + "085" + ], + [ + "9", + "148" + ], + [ + "S", + "abhi" + ], + [ + "Ġ1", + "16" + ], + [ + "Ġ2", + "3" + ], + [ + "à¤Ł", + "स" + ], + [ + "Ġ100", + "50" + ], + [ + "20", + "63" + ], + [ + "à¤ĩ", + "न" + ], + [ + "10", + "21" + ], + [ + "10", + "72" + ], + [ + "11", + "40" + ], + [ + "12", + "27" + ], + [ + "12", + "33" + ], + [ + "13", + "21" + ], + [ + "14", + "06" + ], + [ + "18", + "57" + ], + [ + "18", + "77" + ], + [ + "18", + "24" + ], + [ + "19", + "17" + ], + [ + "19", + "31" + ], + [ + "19", + "46" + ], + [ + "16", + "25" + ], + [ + "22", + "69" + ], + [ + "17", + "92" + ], + [ + "17", + "43" + ], + [ + "Ġ6", + "4" + ], + [ + "ki", + "tabon" + ], + [ + "Ġ8", + "8" + ], + [ + "100", + "8" + ], + [ + "66", + "22" + ], + [ + "66", + "33" + ], + [ + "66", + "41" + ], + [ + "57", + "8" + ], + [ + "57", + "65" + ], + [ + "56", + "00" + ], + [ + "56", + "74" + ], + [ + "93", + "18" + ], + [ + "93", + "08" + ], + [ + "93", + "74" + ], + [ + "93", + "29" + ], + [ + "96", + "20" + ], + [ + "44", + "33" + ], + [ + "92", + "50" + ], + [ + "92", + "58" + ], + [ + "87", + "04" + ], + [ + "27", + "16" + ], + [ + "27", + "89" + ], + [ + "27", + "69" + ], + [ + "49", + "00" + ], + [ + "49", + "57" + ], + [ + "47", + "89" + ], + [ + "47", + "37" + ], + [ + "33", + "10" + ], + [ + "53", + "16" + ], + [ + "Ġpri", + "ces" + ], + [ + "37", + "15" + ], + [ + "37", + "61" + ], + [ + "42", + "11" + ], + [ + "42", + "12" + ], + [ + "42", + "62" + ], + [ + "72", + "13" + ], + [ + "72", + "16" + ], + [ + "72", + "96" + ], + [ + "52", + "93" + ], + [ + "52", + "80" + ], + [ + "58", + "61" + ], + [ + "63", + "06" + ], + [ + "74", + "57" + ], + [ + "74", + "36" + ], + [ + "62", + "63" + ], + [ + "32", + "11" + ], + [ + "32", + "65" + ], + [ + "32", + "84" + ], + [ + "35", + "17" + ], + [ + "79", + "12" + ], + [ + "88", + "35" + ], + [ + "43", + "15" + ], + [ + "75", + "98" + ], + [ + "54", + "13" + ], + [ + "54", + "33" + ], + [ + "38", + "15" + ], + [ + "38", + "65" + ], + [ + "39", + "19" + ], + [ + "95", + "20" + ], + [ + "366", + "6" + ], + [ + "67", + "98" + ], + [ + "86", + "17" + ], + [ + "48", + "28" + ], + [ + "36", + "02" + ], + [ + "603", + "5" + ], + [ + "800", + "8" + ], + [ + "115", + "9" + ], + [ + "296", + "2" + ], + [ + "387", + "1" + ], + [ + "347", + "8" + ], + [ + "518", + "3" + ], + [ + "756", + "3" + ], + [ + "803", + "5" + ], + [ + "844", + "0" + ], + [ + "849", + "7" + ], + [ + "966", + "7" + ], + [ + "663", + "0" + ], + [ + "219", + "2" + ], + [ + "249", + "0" + ], + [ + "247", + "0" + ], + [ + "268", + "3" + ], + [ + "308", + "0" + ], + [ + "437", + "4" + ], + [ + "502", + "8" + ], + [ + "533", + "3" + ], + [ + "627", + "1" + ], + [ + "647", + "3" + ], + [ + "674", + "8" + ], + [ + "709", + "3" + ], + [ + "762", + "6" + ], + [ + "788", + "0" + ], + [ + "816", + "6" + ], + [ + "823", + "6" + ], + [ + "917", + "3" + ], + [ + "949", + "1" + ], + [ + "953", + "8" + ], + [ + "6", + "0" + ], + [ + "7", + "1" + ], + [ + "J", + "o" + ], + [ + "S", + "t" + ], + [ + "s", + "aare" + ], + [ + "Ġ", + "te" + ], + [ + "Ġw", + "ord" + ], + [ + "ude", + "nts" + ], + [ + "10", + "5" + ], + [ + "12", + "1" + ], + [ + "13", + "3" + ], + [ + "13", + "4" + ], + [ + "nsi", + "on" + ], + [ + "Ġany", + "thing" + ], + [ + "Ġjis", + "ka" + ], + [ + "Order", + "s" + ], + [ + "Ticke", + "ts" + ], + [ + "St", + "udents" + ], + [ + "Ġte", + "nsion" + ], + [ + "2", + "16" + ], + [ + "2", + "66" + ], + [ + "2", + "44" + ], + [ + "2", + "69" + ], + [ + "2", + "135" + ], + [ + "2", + "924" + ], + [ + "3", + "22" + ], + [ + "3", + "06" + ], + [ + "3", + "50" + ], + [ + "3", + "04" + ], + [ + "3", + "05" + ], + [ + "3", + "57" + ], + [ + "3", + "68" + ], + [ + "3", + "111" + ], + [ + "3", + "085" + ], + [ + "3", + "139" + ], + [ + "3", + "075" + ], + [ + "3", + "899" + ], + [ + "4", + "202" + ], + [ + "4", + "19" + ], + [ + "4", + "16" + ], + [ + "4", + "02" + ], + [ + "4", + "03" + ], + [ + "4", + "58" + ], + [ + "4", + "68" + ], + [ + "4", + "59" + ], + [ + "4", + "076" + ], + [ + "4", + "116" + ], + [ + "5", + "15" + ], + [ + "5", + "93" + ], + [ + "5", + "96" + ], + [ + "5", + "87" + ], + [ + "5", + "58" + ], + [ + "5", + "135" + ], + [ + "5", + "663" + ], + [ + "5", + "934" + ], + [ + "5", + "924" + ], + [ + "6", + "01" + ], + [ + "6", + "57" + ], + [ + "6", + "56" + ], + [ + "6", + "93" + ], + [ + "6", + "96" + ], + [ + "6", + "53" + ], + [ + "7", + "6" + ], + [ + "7", + "8" + ], + [ + "7", + "06" + ], + [ + "7", + "03" + ], + [ + "7", + "57" + ], + [ + "7", + "92" + ], + [ + "7", + "87" + ], + [ + "7", + "33" + ], + [ + "7", + "68" + ], + [ + "7", + "32" + ], + [ + "7", + "35" + ], + [ + "7", + "69" + ], + [ + "7", + "085" + ], + [ + "8", + "15" + ], + [ + "8", + "18" + ], + [ + "8", + "09" + ], + [ + "8", + "57" + ], + [ + "8", + "87" + ], + [ + "8", + "27" + ], + [ + "8", + "37" + ], + [ + "8", + "59" + ], + [ + "8", + "24" + ], + [ + "8", + "075" + ], + [ + "8", + "564" + ], + [ + "8", + "076" + ], + [ + "8", + "899" + ], + [ + "8", + "125" + ], + [ + "9", + "10" + ], + [ + "9", + "05" + ], + [ + "9", + "09" + ], + [ + "9", + "57" + ], + [ + "9", + "79" + ], + [ + "9", + "075" + ], + [ + "9", + "142" + ], + [ + "G", + "rab" + ], + [ + "V", + "is" + ], + [ + "Ġ1", + "8" + ], + [ + "Ġ1", + "15" + ], + [ + "Ġ1", + "12" + ], + [ + "Ġ1", + "23" + ], + [ + "Ġ2", + "4" + ], + [ + "ĠC", + "an" + ], + [ + "15", + "93" + ], + [ + "15", + "68" + ], + [ + "15", + "74" + ], + [ + "15", + "24" + ], + [ + "15", + "86" + ], + [ + "15", + "48" + ], + [ + "Ġ3", + "4" + ], + [ + "20", + "42" + ], + [ + "20", + "73" + ], + [ + "20", + "80" + ], + [ + "20", + "70" + ], + [ + "11", + "63" + ], + [ + "12", + "6" + ], + [ + "12", + "68" + ], + [ + "12", + "25" + ], + [ + "13", + "7" + ], + [ + "13", + "19" + ], + [ + "13", + "92" + ], + [ + "13", + "95" + ], + [ + "14", + "4" + ], + [ + "14", + "5" + ], + [ + "14", + "7" + ], + [ + "14", + "29" + ], + [ + "18", + "3" + ], + [ + "18", + "7" + ], + [ + "18", + "18" + ], + [ + "18", + "79" + ], + [ + "18", + "67" + ], + [ + "18", + "99" + ], + [ + "19", + "64" + ], + [ + "19", + "25" + ], + [ + "16", + "12" + ], + [ + "16", + "16" + ], + [ + "22", + "20" + ], + [ + "22", + "77" + ], + [ + "17", + "14" + ], + [ + "Ġ7", + "2" + ], + [ + "Ġ9", + "4" + ], + [ + "Ġ9", + "5" + ], + [ + "Ġ4", + "1" + ], + [ + "Ġ12", + "5" + ], + [ + "Ġ8", + "2" + ], + [ + "Ġ8", + "3" + ], + [ + "Ġ8", + "5" + ], + [ + "50", + "33" + ], + [ + "Ġ14", + "0" + ], + [ + "Ġchho", + "te" + ], + [ + "66", + "57" + ], + [ + "66", + "31" + ], + [ + "66", + "28" + ], + [ + "57", + "14" + ], + [ + "57", + "37" + ], + [ + "57", + "52" + ], + [ + "56", + "10" + ], + [ + "56", + "57" + ], + [ + "56", + "44" + ], + [ + "56", + "58" + ], + [ + "56", + "25" + ], + [ + "93", + "89" + ], + [ + "93", + "94" + ], + [ + "96", + "10" + ], + [ + "96", + "21" + ], + [ + "96", + "90" + ], + [ + "96", + "55" + ], + [ + "44", + "14" + ], + [ + "44", + "74" + ], + [ + "44", + "65" + ], + [ + "92", + "6" + ], + [ + "92", + "89" + ], + [ + "92", + "42" + ], + [ + "92", + "79" + ], + [ + "89", + "21" + ], + [ + "89", + "89" + ], + [ + "87", + "75" + ], + [ + "27", + "3" + ], + [ + "27", + "90" + ], + [ + "27", + "85" + ], + [ + "49", + "02" + ], + [ + "49", + "98" + ], + [ + "49", + "42" + ], + [ + "49", + "78" + ], + [ + "47", + "96" + ], + [ + "47", + "64" + ], + [ + "47", + "95" + ], + [ + "33", + "02" + ], + [ + "33", + "29" + ], + [ + "98", + "21" + ], + [ + "98", + "53" + ], + [ + "98", + "68" + ], + [ + "98", + "65" + ], + [ + "53", + "13" + ], + [ + "53", + "91" + ], + [ + "53", + "24" + ], + [ + "53", + "55" + ], + [ + "37", + "18" + ], + [ + "37", + "21" + ], + [ + "37", + "64" + ], + [ + "42", + "98" + ], + [ + "72", + "00" + ], + [ + "72", + "49" + ], + [ + "72", + "91" + ], + [ + "52", + "87" + ], + [ + "68", + "18" + ], + [ + "68", + "64" + ], + [ + "63", + "1" + ], + [ + "63", + "05" + ], + [ + "74", + "58" + ], + [ + "à¤Ĩ", + "त" + ], + [ + "62", + "93" + ], + [ + "62", + "44" + ], + [ + "62", + "89" + ], + [ + "77", + "72" + ], + [ + "77", + "79" + ], + [ + "77", + "75" + ], + [ + "32", + "44" + ], + [ + "32", + "86" + ], + [ + "35", + "66" + ], + [ + "65", + "10" + ], + [ + "79", + "94" + ], + [ + "69", + "44" + ], + [ + "69", + "98" + ], + [ + "59", + "01" + ], + [ + "59", + "87" + ], + [ + "59", + "47" + ], + [ + "Ġmention", + "s" + ], + [ + "88", + "23" + ], + [ + "88", + "87" + ], + [ + "88", + "81" + ], + [ + "43", + "18" + ], + [ + "43", + "47" + ], + [ + "43", + "94" + ], + [ + "29", + "22" + ], + [ + "29", + "58" + ], + [ + "40", + "18" + ], + [ + "40", + "50" + ], + [ + "75", + "49" + ], + [ + "54", + "04" + ], + [ + "54", + "56" + ], + [ + "64", + "09" + ], + [ + "73", + "92" + ], + [ + "73", + "40" + ], + [ + "38", + "23" + ], + [ + "84", + "23" + ], + [ + "84", + "80" + ], + [ + "61", + "20" + ], + [ + "39", + "07" + ], + [ + "95", + "27" + ], + [ + "366", + "8" + ], + [ + "46", + "89" + ], + [ + "67", + "44" + ], + [ + "28", + "15" + ], + [ + "86", + "75" + ], + [ + "97", + "98" + ], + [ + "97", + "85" + ], + [ + "70", + "16" + ], + [ + "99", + "15" + ], + [ + "603", + "4" + ], + [ + "603", + "9" + ], + [ + "800", + "4" + ], + [ + "996", + "1" + ], + [ + "996", + "5" + ], + [ + "996", + "9" + ], + [ + "115", + "6" + ], + [ + "Ġwho", + "se" + ], + [ + "296", + "5" + ], + [ + "387", + "2" + ], + [ + "347", + "4" + ], + [ + "518", + "4" + ], + [ + "789", + "1" + ], + [ + "803", + "2" + ], + [ + "844", + "8" + ], + [ + "966", + "0" + ], + [ + "414", + "5" + ], + [ + "414", + "6" + ], + [ + "453", + "1" + ], + [ + "453", + "6" + ], + [ + "600", + "3" + ], + [ + "600", + "9" + ], + [ + "606", + "1" + ], + [ + "722", + "7" + ], + [ + "944", + "3" + ], + [ + "944", + "9" + ], + [ + "972", + "5" + ], + [ + "972", + "8" + ], + [ + "Ġhon", + "a" + ], + [ + "34", + "35" + ], + [ + "83", + "12" + ], + [ + "216", + "9" + ], + [ + "266", + "6" + ], + [ + "244", + "8" + ], + [ + "269", + "7" + ], + [ + "322", + "8" + ], + [ + "306", + "4" + ], + [ + "350", + "7" + ], + [ + "304", + "3" + ], + [ + "305", + "9" + ], + [ + "357", + "7" + ], + [ + "368", + "6" + ], + [ + "419", + "5" + ], + [ + "416", + "5" + ], + [ + "402", + "9" + ], + [ + "403", + "7" + ], + [ + "458", + "6" + ], + [ + "468", + "1" + ], + [ + "459", + "9" + ], + [ + "515", + "8" + ], + [ + "593", + "9" + ], + [ + "596", + "9" + ], + [ + "587", + "5" + ], + [ + "558", + "3" + ], + [ + "657", + "4" + ], + [ + "656", + "9" + ], + [ + "693", + "5" + ], + [ + "696", + "8" + ], + [ + "653", + "4" + ], + [ + "706", + "7" + ], + [ + "703", + "4" + ], + [ + "757", + "0" + ], + [ + "792", + "9" + ], + [ + "787", + "1" + ], + [ + "733", + "6" + ], + [ + "768", + "5" + ], + [ + "732", + "6" + ], + [ + "769", + "1" + ], + [ + "815", + "1" + ], + [ + "818", + "9" + ], + [ + "809", + "4" + ], + [ + "857", + "5" + ], + [ + "887", + "3" + ], + [ + "827", + "4" + ], + [ + "837", + "8" + ], + [ + "859", + "7" + ], + [ + "824", + "1" + ], + [ + "910", + "8" + ], + [ + "905", + "7" + ], + [ + "909", + "1" + ], + [ + "957", + "6" + ], + [ + "979", + "0" + ], + [ + "Vis", + "its" + ], + [ + "S", + "aare" + ], + [ + "re", + "ation" + ], + [ + "me", + "re" + ], + [ + "Ġa", + "mou" + ], + [ + "Ġc", + "reation" + ], + [ + "Ġt", + "wo" + ], + [ + "Ġe", + "xpens" + ], + [ + "Ġe", + "ve" + ], + [ + "à¥ĭ", + "?" + ], + [ + "Ġth", + "ing" + ], + [ + "Ġwal", + "a" + ], + [ + "Ġstar", + "ted" + ], + [ + "Ġgay", + "e" + ], + [ + "Ġbounda", + "ries" + ], + [ + "Ġkaun", + "se" + ], + [ + "Ġwor", + "ks" + ], + [ + "Ġamou", + "nts" + ], + [ + "Ġexpens", + "ive" + ], + [ + "Ġeve", + "n" + ], + [ + "P", + "ayment" + ], + [ + "s", + "abhi" + ], + [ + "Ġc", + "at" + ], + [ + "Ġs", + "ki" + ], + [ + "Ġre", + "al" + ], + [ + "Ġde", + "pt" + ], + [ + "ch", + "es" + ], + [ + "Ġstatus", + "es" + ], + [ + "ĠB", + "as" + ], + [ + "à¤ĩ", + "à¤ľ" + ], + [ + "Ġgo", + "es" + ], + [ + "Ac", + "counts" + ], + [ + "Ġcat", + "ches" + ], + [ + "'", + "m" + ], + [ + "S", + "hipments" + ], + [ + "a", + "pes" + ], + [ + "d", + "um" + ], + [ + "Ġ", + "che" + ], + [ + "Ġ", + "tal" + ], + [ + "Ġ", + "elect" + ], + [ + "Ġc", + "reate" + ], + [ + "Ġc", + "are" + ], + [ + "र", + "फ" + ], + [ + "न", + "à¤ĸ" + ], + [ + "arch", + "on" + ], + [ + "Ġà¤ħ", + "द" + ], + [ + "Ġत", + "रफ" + ], + [ + "Ġत", + "नà¤ĸ" + ], + [ + "Ġho", + "ga" + ], + [ + "à¤ħ", + "र" + ], + [ + "Ġski", + "p" + ], + [ + "apes", + "t" + ], + [ + "Ġche", + "apest" + ], + [ + "Ġtal", + "ly" + ], + [ + "Ġelect", + "ronics" + ], + [ + ".", + "." + ], + [ + "1", + "10" + ], + [ + "3", + "0" + ], + [ + "D", + "ik" + ], + [ + "I", + "n" + ], + [ + "K", + "h" + ], + [ + "k", + "o" + ], + [ + "s", + "o" + ], + [ + "ĺ", + "à¤Ł" + ], + [ + "Ġà¤", + "ĺà¤Ł" + ], + [ + "Ġs", + "ubs" + ], + [ + "Ġb", + "ani" + ], + [ + "ĠM", + "ake" + ], + [ + "Ġk", + "in" + ], + [ + "Ġà¤ķ", + "स" + ], + [ + "Ġà¤ķ", + "à¤ł" + ], + [ + "Ġd", + "in" + ], + [ + "Ġd", + "ow" + ], + [ + "ल", + "à¤ķर" + ], + [ + "Ġcost", + "s" + ], + [ + "Ġcurrent", + "ly" + ], + [ + "Ġà¤ħ", + "म" + ], + [ + "Ġnull", + "s" + ], + [ + "11", + "2" + ], + [ + "Ġas", + "c" + ], + [ + "Ġपर", + "व" + ], + [ + "12", + "3" + ], + [ + "14", + "0" + ], + [ + "Ġal", + "so" + ], + [ + "Ġusk", + "a" + ], + [ + "De", + "vices" + ], + [ + "ek", + "dum" + ], + [ + "Ġbana", + "ye" + ], + [ + "Ġà¤īत", + "रत" + ], + [ + "à¤Łà¤®", + "र" + ], + [ + "Dik", + "hao" + ], + [ + "Kh", + "archon" + ], + [ + "Ġà¤ĺà¤Ł", + "त" + ], + [ + "Ġdow", + "n" + ], + [ + "E", + "k" + ], + [ + "P", + "ayments" + ], + [ + "S", + "ubscriptions" + ], + [ + "p", + "pe" + ], + [ + "r", + "ked" + ], + [ + "t", + "te" + ], + [ + "y", + "be" + ], + [ + "Ġ", + "quan" + ], + [ + "ti", + "ties" + ], + [ + "Ġa", + "ppe" + ], + [ + "up", + "per" + ], + [ + "Ġs", + "i" + ], + [ + "Ġs", + "as" + ], + [ + "Ġw", + "ell" + ], + [ + "Ġe", + "ar" + ], + [ + "Ġn", + "orth" + ], + [ + "Ġde", + "na" + ], + [ + "र", + "प" + ], + [ + "र", + "à¤ľ" + ], + [ + "Ġस", + "स" + ], + [ + "Ġस", + "à¤Ł" + ], + [ + "Ġj", + "aye" + ], + [ + "ĠN", + "eed" + ], + [ + "Ġma", + "rked" + ], + [ + "Ġma", + "ybe" + ], + [ + "à¥ĭà¤Ĥ", + "," + ], + [ + "Ġun", + "ko" + ], + [ + "Ġà¤Ĩ", + "à¤ĸ" + ], + [ + "Ġtotal", + "s" + ], + [ + "Ġle", + "na" + ], + [ + "Ġत", + "ब" + ], + [ + "Ġhis", + "ab" + ], + [ + "Ġà¤īप", + "à¤ķरण" + ], + [ + "Ġye", + "ah" + ], + [ + "Ġwri", + "tte" + ], + [ + "à¤ī", + "त" + ], + [ + "Ġign", + "ores" + ], + [ + "Ġso", + "me" + ], + [ + "Ġà¤Ń", + "र" + ], + [ + "Ġam", + "ong" + ], + [ + "Ġरà¤ĸ", + "à¤ķर" + ], + [ + "Ġaay", + "e" + ], + [ + "Ġti", + "mes" + ], + [ + "Ticke", + "t" + ], + [ + "à¤ĩन", + "व" + ], + [ + "ĠBas", + "ically" + ], + [ + "Ġquan", + "tities" + ], + [ + "Ġappe", + "ar" + ], + [ + "Ġwritte", + "n" + ], + [ + "E", + "xpenses" + ], + [ + "S", + "are" + ], + [ + "S", + "hipment" + ], + [ + "T", + "H" + ], + [ + "b", + "ounda" + ], + [ + "i", + "x" + ], + [ + "j", + "o" + ], + [ + "l", + "hi" + ], + [ + "l", + "ow" + ], + [ + "n", + "ly" + ], + [ + "o", + "od" + ], + [ + "p", + "ayment" + ], + [ + "s", + "are" + ], + [ + "w", + "a" + ], + [ + "à", + "¦" + ], + [ + "pe", + "at" + ], + [ + "in", + "ta" + ], + [ + "in", + "voice" + ], + [ + "id", + "es" + ], + [ + "ma", + "ine" + ], + [ + "Ġa", + "a" + ], + [ + "Ġa", + "ana" + ], + [ + "Ġa", + "ise" + ], + [ + "|>", + "'" + ], + [ + "Ġc", + "ould" + ], + [ + "ĠI", + "t" + ], + [ + "Pro", + "ducts" + ], + [ + "um", + "p" + ], + [ + "ay", + "s" + ], + [ + "Ġs", + "outh" + ], + [ + "Ġs", + "ides" + ], + [ + "Ġs", + "ays" + ], + [ + "Ġre", + "peat" + ], + [ + "Ġis", + "nt" + ], + [ + "Ġm", + "ix" + ], + [ + "ate", + "ver" + ], + [ + "Ġb", + "alan" + ], + [ + "Ġe", + "m" + ], + [ + "ĠM", + "ujhe" + ], + [ + "Ġde", + "lhi" + ], + [ + "ok", + "ar" + ], + [ + "Ġf", + "ara" + ], + [ + "Ġd", + "ump" + ], + [ + "Ġस", + "à¤ĸ" + ], + [ + "à¥Ģ", + "à¤ģ" + ], + [ + "Ġme", + "hen" + ], + [ + "ल", + "त" + ], + [ + "म", + "ह" + ], + [ + "ĠO", + "nly" + ], + [ + "Ġj", + "ab" + ], + [ + "प", + "à¤ķरण" + ], + [ + "Ġन", + "ल" + ], + [ + "Ġma", + "uj" + ], + [ + "Ġà¤ī", + "तर" + ], + [ + "à¤ļ", + "ढ" + ], + [ + "Ġh", + "okar" + ], + [ + "à¤ľ", + "ब" + ], + [ + "Ġla", + "na" + ], + [ + "Ġle", + "ke" + ], + [ + "Ġjin", + "he" + ], + [ + "Ġba", + "at" + ], + [ + "Ġba", + "aki" + ], + [ + "Ġफ", + "र" + ], + [ + "Ġjink", + "ay" + ], + [ + "Ġpeh", + "li" + ], + [ + "Ġkauns", + "a" + ], + [ + "Ġlin", + "k" + ], + [ + "Ġà¤ĩस", + "स" + ], + [ + "Ġusk", + "i" + ], + [ + "à¤ī", + "पà¤ķरण" + ], + [ + "Ġch", + "inta" + ], + [ + "Ġany", + "one" + ], + [ + "Ġhu", + "ye" + ], + [ + "Ġdikh", + "wa" + ], + [ + "Ġpu", + "ri" + ], + [ + "Ġpo", + "ore" + ], + [ + "Ġwh", + "atever" + ], + [ + "Ġjinki", + "i" + ], + [ + "kind", + "s" + ], + [ + "Ġcho", + "ti" + ], + [ + "li", + "ke" + ], + [ + "à¤Ĩ", + "प" + ], + [ + "Ġresul", + "t" + ], + [ + "ner", + "s" + ], + [ + "..", + "." + ], + [ + "In", + "voices" + ], + [ + "In", + "voice" + ], + [ + "upper", + "case" + ], + [ + "Ġsas", + "ti" + ], + [ + "Ġear", + "ners" + ], + [ + "bounda", + "ry" + ], + [ + "low", + "er" + ], + [ + "Ġmix", + "ed" + ], + [ + "Ġbalan", + "ces" + ], + [ + "Ġfara", + "k" + ], + [ + "Ġmauj", + "ood" + ] + ] + } +} \ No newline at end of file