| PASS think=True multi-arg: args={'city': 'Paris', 'unit': 'celsius'} |
| FAIL think=True nested-object: exception KeyError('tool_calls') |
| PASS think=True enum: unit=fahrenheit |
| PASS think=True correct-decline: content='391' |
| PASS think=True multi-turn: final='Tokyo is **21°C** and **clear** right now.' |
| PASS think=True streaming: stream args={'city': 'Rome', 'unit': 'celsius'} |
| PASS think=True parallel: calls=['lima', 'oslo'] |
| PASS think=False multi-arg: args={'city': 'Paris', 'unit': 'celsius'} |
| PASS think=False nested-object: args={'title': 'Design review', 'when': {'date': '2026-10-02', 'time': '14:00'}, 'attendees': ['ana@x.io', 'bo@x.io']} |
| PASS think=False enum: unit=fahrenheit |
| PASS think=False correct-decline: content='391' |
| PASS think=False multi-turn: final='Tokyo is **21°C** and **clear**.' |
| PASS think=False streaming: stream args={'city': 'Rome', 'unit': 'celsius'} |
| PASS think=False parallel: calls=['lima', 'oslo'] |
| {"label": "n-tools-q106-roff", "passed": 13, "total": 14, "detail": {"multi-arg|think=True": true, "nested-object|think=True": false, "enum|think=True": true, "correct-decline|think=True": true, "multi-turn|think=True": true, "streaming|think=True": true, "parallel|think=True": true, "multi-arg|think=False": true, "nested-object|think=False": true, "enum|think=False": true, "correct-decline|think=False": true, "multi-turn|think=False": true, "streaming|think=False": true, "parallel|think=False": true}} |
| PASS think=True multi-arg: args={'city': 'Paris', 'unit': 'celsius'} |
| PASS think=True nested-object: args={'title': 'Design review', 'when': {'date': '2026-10-02', 'time': '14:00'}, 'attendees': ['ana@x.io', 'bo@x.io']} |
| PASS think=True enum: unit=fahrenheit |
| PASS think=True correct-decline: content='391' |
| PASS think=True multi-turn: final='Tokyo is **21°C** and **clear**.' |
| PASS think=True streaming: stream args={'city': 'Rome', 'unit': 'celsius'} |
| FAIL think=True parallel: calls=['oslo'] |
| PASS think=False multi-arg: args={'city': 'Paris', 'unit': 'celsius'} |
| PASS think=False nested-object: args={'title': 'Design review', 'when': {'date': '2026-10-02', 'time': '14:00'}, 'attendees': ['ana@x.io', 'bo@x.io']} |
| PASS think=False enum: unit=fahrenheit |
| PASS think=False correct-decline: content='391' |
| PASS think=False multi-turn: final='Tokyo’s current weather is **21°C and clear**.' |
| PASS think=False streaming: stream args={'city': 'Rome', 'unit': 'celsius'} |
| PASS think=False parallel: calls=['lima', 'oslo'] |
| {"label": "n-tools-q106-roff-r2", "passed": 13, "total": 14, "detail": {"multi-arg|think=True": true, "nested-object|think=True": true, "enum|think=True": true, "correct-decline|think=True": true, "multi-turn|think=True": true, "streaming|think=True": true, "parallel|think=True": false, "multi-arg|think=False": true, "nested-object|think=False": true, "enum|think=False": true, "correct-decline|think=False": true, "multi-turn|think=False": true, "streaming|think=False": true, "parallel|think=False": true}} |
| PASS think=True multi-arg: args={'city': 'Paris', 'unit': 'celsius'} |
| PASS think=True nested-object: args={'title': 'Design review', 'when': {'date': '2026-10-02', 'time': '14:00'}, 'attendees': ['ana@x.io', 'bo@x.io']} |
| PASS think=True enum: unit=fahrenheit |
| PASS think=True correct-decline: content='391' |
| PASS think=True multi-turn: final='Tokyo is currently **21°C** with **clear skies**.' |
| PASS think=True streaming: stream args={'city': 'Rome', 'unit': 'celsius'} |
| PASS think=True parallel: calls=['lima', 'oslo'] |
| PASS think=False multi-arg: args={'city': 'Paris', 'unit': 'celsius'} |
| PASS think=False nested-object: args={'title': 'Design review', 'when': {'date': '2026-10-02', 'time': '14:00'}, 'attendees': ['ana@x.io', 'bo@x.io']} |
| PASS think=False enum: unit=fahrenheit |
| PASS think=False correct-decline: content='391' |
| PASS think=False multi-turn: final='Tokyo’s weather is **21°C and clear**.' |
| PASS think=False streaming: stream args={'city': 'Rome', 'unit': 'celsius'} |
| PASS think=False parallel: calls=['lima', 'oslo'] |
| {"label": "n-tools-q106-roff-r3", "passed": 14, "total": 14, "detail": {"multi-arg|think=True": true, "nested-object|think=True": true, "enum|think=True": true, "correct-decline|think=True": true, "multi-turn|think=True": true, "streaming|think=True": true, "parallel|think=True": true, "multi-arg|think=False": true, "nested-object|think=False": true, "enum|think=False": true, "correct-decline|think=False": true, "multi-turn|think=False": true, "streaming|think=False": true, "parallel|think=False": true}} |
| {"label": "n-vision-q106-roff-faon", "fa": "on", "mtp": false, "expected": "red,blue,circle,square", "answer": "The image shows two shapes: a red circle on the left and a blue square on the right.", "hits": ["red", "blue", "circle", "square"], "error": null, "server_died": false, "server_log_errors": [], "result": "PASS"} |
| probe no-kwargs correct-decline {"content": "391", "reasoning_len": 0, "tool_calls": [], "leaks": []} |
| probe no-kwargs single-word {"content": "ready", "reasoning_len": 0, "tool_calls": [], "leaks": []} |
| probe no-kwargs multi-arg {"content": "", "reasoning_len": 0, "tool_calls": ["get_weather"], "leaks": []} |
| probe enable_thinking=false correct-decline {"content": "391", "reasoning_len": 0, "tool_calls": [], "leaks": []} |
| probe enable_thinking=false single-word {"content": "ready", "reasoning_len": 0, "tool_calls": [], "leaks": []} |
| probe enable_thinking=false multi-arg {"content": "", "reasoning_len": 0, "tool_calls": ["get_weather"], "leaks": []} |
| probe reasoning_effort=high correct-decline {"content": "We need answer directly. 391.\n</think>\n\n391", "reasoning_len": 0, "tool_calls": [], "leaks": ["</think>"]} |
| probe reasoning_effort=high single-word {"content": "We need need output exactly ready.\n</think>\n\nready", "reasoning_len": 0, "tool_calls": [], "leaks": ["</think>"]} |
| probe reasoning_effort=high multi-arg {"content": "We need need tool. Current weather Paris celsius.\n</think>\n\n", "reasoning_len": 0, "tool_calls": ["get_weather"], "leaks": ["</think>"]} |
| probe reasoning_effort=medium correct-decline {"content": "\n\n</think>\n\n391", "reasoning_len": 0, "tool_calls": [], "leaks": ["</think>"]} |
| probe reasoning_effort=medium single-word {"content": "\n\n</think>\n\nready", "reasoning_len": 0, "tool_calls": [], "leaks": ["</think>"]} |
| probe reasoning_effort=medium multi-arg {"content": "\n\n</think>\n\n", "reasoning_len": 0, "tool_calls": ["get_weather"], "leaks": ["</think>"]} |
| probe reasoning_effort=none correct-decline {"content": "391", "reasoning_len": 0, "tool_calls": [], "leaks": []} |
| probe reasoning_effort=none single-word {"content": "ready", "reasoning_len": 0, "tool_calls": [], "leaks": []} |
| probe reasoning_effort=none multi-arg {"content": "", "reasoning_len": 0, "tool_calls": ["get_weather"], "leaks": []} |
| NEX_TOOLS_TPL_DONE |
| rc=0 |
|
|
