File size: 876 Bytes
2bc6021 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 | """Official IFBench strict prompt-level scoring in the isolated eval environment."""
import json
import sys
from ifbench import instructions_registry
def score(task, response):
passed = []
for name, kwargs in zip(task["instruction_id_list"], task["kwargs"]):
checker = instructions_registry.INSTRUCTION_DICT[name](name)
checker.build_description(**{k: v for k, v in kwargs.items() if v is not None})
needed = checker.get_instruction_args()
if needed and "prompt" in needed:
checker.build_description(prompt=task["messages"][-1]["content"])
passed.append(bool(response.strip()) and bool(checker.check_following(response)))
return {"correct": all(passed), "instructions": passed}
if __name__ == "__main__":
rows = json.load(sys.stdin)
print(json.dumps([score(r["task"], r["response"]) for r in rows]))
|