User1342 commited on
Commit
2776025
·
1 Parent(s): 04e4eec

Keep the sentence a refused tool wrote, so a finished run says why rather than tool_error

Browse files
distinct_agent/harness.py CHANGED
@@ -1416,6 +1416,16 @@ def _event_from_result(value: Mapping[str, Any]) -> Mapping[str, Any]:
1416
  error = value.get("error")
1417
  if isinstance(error, Mapping):
1418
  event["error_code"] = str(error.get("code", "tool_error"))[:80]
 
 
 
 
 
 
 
 
 
 
1419
  output = value.get("output")
1420
  if isinstance(output, Mapping):
1421
  saved = output.get("artifact_saved")
 
1416
  error = value.get("error")
1417
  if isinstance(error, Mapping):
1418
  event["error_code"] = str(error.get("code", "tool_error"))[:80]
1419
+ # THE CODE ALONE SAYS NOTHING ANYBODY CAN ACT ON.
1420
+ #
1421
+ # The model sees the full result and can correct itself, but the run
1422
+ # record kept only "tool_error", so a person looking at a finished run
1423
+ # -- or at twenty benchmark runs -- could see that a tool had refused
1424
+ # and never why. "line 7 has 4 fields where the header has 5" is a
1425
+ # sentence somebody can fix; "tool_error" is not.
1426
+ message = error.get("message")
1427
+ if isinstance(message, str) and message.strip():
1428
+ event["error_message"] = message.strip()[:300]
1429
  output = value.get("output")
1430
  if isinstance(output, Mapping):
1431
  saved = output.get("artifact_saved")
tests/test_harness_makes_files.py CHANGED
@@ -562,3 +562,30 @@ def test_a_finished_answer_is_not_treated_as_cut_off(tmp_path) -> None:
562
  tmp_path,
563
  )
564
  assert result.text == "all done"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
562
  tmp_path,
563
  )
564
  assert result.text == "all done"
565
+
566
+
567
+ def test_a_refusal_keeps_the_sentence_somebody_can_act_on() -> None:
568
+ """"tool_error" told nobody anything; the message names the line to fix."""
569
+
570
+ from distinct_agent.harness import _event_from_result
571
+
572
+ event = _event_from_result(
573
+ {
574
+ "tool_id": "create_xlsx",
575
+ "version": "1",
576
+ "ok": False,
577
+ "error": {
578
+ "code": "skill_input_error",
579
+ "message": "line 7 has 4 fields where the header has 5",
580
+ },
581
+ }
582
+ )
583
+ assert event["error_code"] == "skill_input_error"
584
+ assert event["error_message"] == "line 7 has 4 fields where the header has 5"
585
+
586
+
587
+ def test_a_refusal_with_no_message_is_simply_a_code() -> None:
588
+ from distinct_agent.harness import _event_from_result
589
+
590
+ event = _event_from_result({"tool_id": "x", "ok": False, "error": {"code": "denied"}})
591
+ assert "error_message" not in event