Keep the sentence a refused tool wrote, so a finished run says why rather than tool_error
Browse files- distinct_agent/harness.py +10 -0
- tests/test_harness_makes_files.py +27 -0
distinct_agent/harness.py
CHANGED
|
@@ -1416,6 +1416,16 @@ def _event_from_result(value: Mapping[str, Any]) -> Mapping[str, Any]:
|
|
| 1416 |
error = value.get("error")
|
| 1417 |
if isinstance(error, Mapping):
|
| 1418 |
event["error_code"] = str(error.get("code", "tool_error"))[:80]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1419 |
output = value.get("output")
|
| 1420 |
if isinstance(output, Mapping):
|
| 1421 |
saved = output.get("artifact_saved")
|
|
|
|
| 1416 |
error = value.get("error")
|
| 1417 |
if isinstance(error, Mapping):
|
| 1418 |
event["error_code"] = str(error.get("code", "tool_error"))[:80]
|
| 1419 |
+
# THE CODE ALONE SAYS NOTHING ANYBODY CAN ACT ON.
|
| 1420 |
+
#
|
| 1421 |
+
# The model sees the full result and can correct itself, but the run
|
| 1422 |
+
# record kept only "tool_error", so a person looking at a finished run
|
| 1423 |
+
# -- or at twenty benchmark runs -- could see that a tool had refused
|
| 1424 |
+
# and never why. "line 7 has 4 fields where the header has 5" is a
|
| 1425 |
+
# sentence somebody can fix; "tool_error" is not.
|
| 1426 |
+
message = error.get("message")
|
| 1427 |
+
if isinstance(message, str) and message.strip():
|
| 1428 |
+
event["error_message"] = message.strip()[:300]
|
| 1429 |
output = value.get("output")
|
| 1430 |
if isinstance(output, Mapping):
|
| 1431 |
saved = output.get("artifact_saved")
|
tests/test_harness_makes_files.py
CHANGED
|
@@ -562,3 +562,30 @@ def test_a_finished_answer_is_not_treated_as_cut_off(tmp_path) -> None:
|
|
| 562 |
tmp_path,
|
| 563 |
)
|
| 564 |
assert result.text == "all done"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 562 |
tmp_path,
|
| 563 |
)
|
| 564 |
assert result.text == "all done"
|
| 565 |
+
|
| 566 |
+
|
| 567 |
+
def test_a_refusal_keeps_the_sentence_somebody_can_act_on() -> None:
|
| 568 |
+
""""tool_error" told nobody anything; the message names the line to fix."""
|
| 569 |
+
|
| 570 |
+
from distinct_agent.harness import _event_from_result
|
| 571 |
+
|
| 572 |
+
event = _event_from_result(
|
| 573 |
+
{
|
| 574 |
+
"tool_id": "create_xlsx",
|
| 575 |
+
"version": "1",
|
| 576 |
+
"ok": False,
|
| 577 |
+
"error": {
|
| 578 |
+
"code": "skill_input_error",
|
| 579 |
+
"message": "line 7 has 4 fields where the header has 5",
|
| 580 |
+
},
|
| 581 |
+
}
|
| 582 |
+
)
|
| 583 |
+
assert event["error_code"] == "skill_input_error"
|
| 584 |
+
assert event["error_message"] == "line 7 has 4 fields where the header has 5"
|
| 585 |
+
|
| 586 |
+
|
| 587 |
+
def test_a_refusal_with_no_message_is_simply_a_code() -> None:
|
| 588 |
+
from distinct_agent.harness import _event_from_result
|
| 589 |
+
|
| 590 |
+
event = _event_from_result({"tool_id": "x", "ok": False, "error": {"code": "denied"}})
|
| 591 |
+
assert "error_message" not in event
|