| { |
| "title": "Frozen80 subset composition and per-task comparison", |
| "created_date": "2026-09-21", |
| "panel": { |
| "records": 80, |
| "parent_groups": 79, |
| "split": "development regression, not sealed final", |
| "score_labels": "16 No / 0 Yes" |
| }, |
| "models": [ |
| "APUS-OpenJev 4B", |
| "APUS-OpenJev 9B", |
| "Jev official API", |
| "Laya typed / full input" |
| ], |
| "model_totals": { |
| "APUS-OpenJev 4B": { |
| "correct": 66, |
| "total": 80, |
| "percentage": 82.5 |
| }, |
| "APUS-OpenJev 9B": { |
| "correct": 68, |
| "total": 80, |
| "percentage": 85.0 |
| }, |
| "Jev official API": { |
| "correct": 66, |
| "total": 80, |
| "percentage": 82.5 |
| }, |
| "Laya typed / full input": { |
| "correct": 55, |
| "total": 80, |
| "percentage": 68.75 |
| } |
| }, |
| "subsets": [ |
| { |
| "dataset": "browser.dev.records.jsonl", |
| "name": "Browser", |
| "source": "OSU NLP / Mind2Web", |
| "target_capability": "Select the next browser action from page context and a goal.", |
| "records": 16, |
| "groups": 16, |
| "percentage_of_panel": 20.0, |
| "scores": { |
| "APUS-OpenJev 4B": { |
| "correct": 14, |
| "total": 16, |
| "percentage": 87.5 |
| }, |
| "APUS-OpenJev 9B": { |
| "correct": 13, |
| "total": 16, |
| "percentage": 81.25 |
| }, |
| "Jev official API": { |
| "correct": 11, |
| "total": 16, |
| "percentage": 68.75 |
| }, |
| "Laya typed / full input": { |
| "correct": 4, |
| "total": 16, |
| "percentage": 25.0 |
| } |
| } |
| }, |
| { |
| "dataset": "hs3.dev.records.jsonl", |
| "name": "HelpSteer3", |
| "source": "NVIDIA / principle-based judgments", |
| "target_capability": "Judge content against a supplied natural-language principle.", |
| "records": 16, |
| "groups": 16, |
| "percentage_of_panel": 20.0, |
| "scores": { |
| "APUS-OpenJev 4B": { |
| "correct": 12, |
| "total": 16, |
| "percentage": 75.0 |
| }, |
| "APUS-OpenJev 9B": { |
| "correct": 13, |
| "total": 16, |
| "percentage": 81.25 |
| }, |
| "Jev official API": { |
| "correct": 12, |
| "total": 16, |
| "percentage": 75.0 |
| }, |
| "Laya typed / full input": { |
| "correct": 11, |
| "total": 16, |
| "percentage": 68.75 |
| } |
| } |
| }, |
| { |
| "dataset": "boolq.dev.records.jsonl", |
| "name": "BoolQ", |
| "source": "Boolean Questions", |
| "target_capability": "Answer yes/no questions using supporting context.", |
| "records": 16, |
| "groups": 16, |
| "percentage_of_panel": 20.0, |
| "scores": { |
| "APUS-OpenJev 4B": { |
| "correct": 16, |
| "total": 16, |
| "percentage": 100.0 |
| }, |
| "APUS-OpenJev 9B": { |
| "correct": 16, |
| "total": 16, |
| "percentage": 100.0 |
| }, |
| "Jev official API": { |
| "correct": 16, |
| "total": 16, |
| "percentage": 100.0 |
| }, |
| "Laya typed / full input": { |
| "correct": 15, |
| "total": 16, |
| "percentage": 93.75 |
| } |
| } |
| }, |
| { |
| "dataset": "mnli.dev.records.jsonl", |
| "name": "MNLI", |
| "source": "Multi-Genre NLI", |
| "target_capability": "Identify entailment, contradiction or a neutral relationship.", |
| "records": 16, |
| "groups": 16, |
| "percentage_of_panel": 20.0, |
| "scores": { |
| "APUS-OpenJev 4B": { |
| "correct": 11, |
| "total": 16, |
| "percentage": 68.75 |
| }, |
| "APUS-OpenJev 9B": { |
| "correct": 13, |
| "total": 16, |
| "percentage": 81.25 |
| }, |
| "Jev official API": { |
| "correct": 13, |
| "total": 16, |
| "percentage": 81.25 |
| }, |
| "Laya typed / full input": { |
| "correct": 11, |
| "total": 16, |
| "percentage": 68.75 |
| } |
| } |
| }, |
| { |
| "dataset": "score.dev.records.jsonl", |
| "name": "Score", |
| "source": "Google / GoEmotions", |
| "target_capability": "Judge whether a specified emotion attribute is present.", |
| "records": 16, |
| "groups": 15, |
| "percentage_of_panel": 20.0, |
| "scores": { |
| "APUS-OpenJev 4B": { |
| "correct": 13, |
| "total": 16, |
| "percentage": 81.25 |
| }, |
| "APUS-OpenJev 9B": { |
| "correct": 13, |
| "total": 16, |
| "percentage": 81.25 |
| }, |
| "Jev official API": { |
| "correct": 14, |
| "total": 16, |
| "percentage": 87.5 |
| }, |
| "Laya typed / full input": { |
| "correct": 14, |
| "total": 16, |
| "percentage": 87.5 |
| } |
| } |
| } |
| ], |
| "verification": { |
| "all_model_exact_ID_sets_match": true, |
| "all_gold_labels_match": true, |
| "laya_repeats": 3, |
| "laya_identical_predictions": true, |
| "laya_truncated_records": 0, |
| "laya_raw_choices_reparsed": 240, |
| "one_answer_percentage_points": 6.25 |
| }, |
| "sources": { |
| "APUS-OpenJev 4B": { |
| "project_relative_path": "docs/feat-jev-combined/openjet-release-20260921/merged-release/evidence/4B/after.jsonl", |
| "sha256": "ac22ede19c6ade99388038292644df50e9891dff162acdb10029bb3bd343537e" |
| }, |
| "APUS-OpenJev 9B": { |
| "project_relative_path": "docs/feat-jev-combined/openjet-release-20260921/merged-release/evidence/9B/after.jsonl", |
| "sha256": "3f01a868d4fba91a8a1749df37a87f10f0874823eec5a978291c93231773c365" |
| }, |
| "Jev official API": { |
| "project_relative_path": "docs/feat-jev-combined/evidence/9b-checkpoints80/3000/predictions.jsonl", |
| "sha256": "0997f100fd2da6bed882a70c1e5fe7f1d47c8db9c8d79c76bddbf641ce00344d" |
| }, |
| "Laya repeat 1": { |
| "project_relative_path": "docs/feat-jev-combined/evidence/laya-repeated-20260921/round1-typed-fullinput/predictions.jsonl", |
| "sha256": "abdddedaeb337dbde5f5ffba8205ab14bc3a6d831ddce8a7feca4a97bb9b5b12" |
| }, |
| "Laya repeat 2": { |
| "project_relative_path": "docs/feat-jev-combined/evidence/laya-repeated-20260921/round2-typed-fullinput/predictions.jsonl", |
| "sha256": "51556848bf798a0fb6f891be400f60c6b8da3505445795ce31d8f60dd2a56a18" |
| }, |
| "Laya repeat 3": { |
| "project_relative_path": "docs/feat-jev-combined/evidence/laya-repeated-20260921/round3-typed-fullinput/predictions.jsonl", |
| "sha256": "36c637ed48fbcb9eba34494c73bab23d7a539b745b63beee901d8736abfff450" |
| } |
| }, |
| "record_audit": [ |
| { |
| "record_id": "boolq/11b00c8dcb82dfbdaf02aa139e8c70603fb8c432e0fc51873ca00b20815a90f5", |
| "group_id": "evidence-group-83d16ec1b2a65c49a953cfc8b850422af8beb2758d72f39b23cf3e022be597a1", |
| "dataset": "boolq.dev.records.jsonl", |
| "gold": "yes", |
| "predictions": { |
| "APUS-OpenJev 4B": "yes", |
| "APUS-OpenJev 9B": "yes", |
| "Jev official API": "yes", |
| "Laya typed / full input": "yes" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "boolq/155bc4f5246c5b35ec14d7a2f697381de04c2cefbbc293b387bfdf83f2fea269", |
| "group_id": "evidence-group-9984381b97b26a3aa599aab8033350847b8b1cd7f6e2ed31ac1f8e954228e734", |
| "dataset": "boolq.dev.records.jsonl", |
| "gold": "yes", |
| "predictions": { |
| "APUS-OpenJev 4B": "yes", |
| "APUS-OpenJev 9B": "yes", |
| "Jev official API": "yes", |
| "Laya typed / full input": "yes" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "boolq/179184609853d9f8f1f862bd5c3113e363f622606fcf5c188cf0da1d3ed61e86", |
| "group_id": "evidence-group-afe52b84e10e464fe71423d120372724c1bb668d7dc611953a6946c3968c9b29", |
| "dataset": "boolq.dev.records.jsonl", |
| "gold": "yes", |
| "predictions": { |
| "APUS-OpenJev 4B": "yes", |
| "APUS-OpenJev 9B": "yes", |
| "Jev official API": "yes", |
| "Laya typed / full input": "yes" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "boolq/31fa2124dc5769d9e6fae53be4190290131f8fb025a089f8372d6f5d61504af5", |
| "group_id": "evidence-group-47ad7ea2eb35ffe5d68eebc068d2c2dc4668d58316bc11a90870609bd735fa77", |
| "dataset": "boolq.dev.records.jsonl", |
| "gold": "no", |
| "predictions": { |
| "APUS-OpenJev 4B": "no", |
| "APUS-OpenJev 9B": "no", |
| "Jev official API": "no", |
| "Laya typed / full input": "no" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "boolq/3f87d9cb171dfe78b389d22677512f92cb48f7eedeefd35f2db1a2444cf08af2", |
| "group_id": "evidence-group-0ffadfb3889de88a9be6abacda9f33200265f55f988f8f4e30d679a6abb60220", |
| "dataset": "boolq.dev.records.jsonl", |
| "gold": "yes", |
| "predictions": { |
| "APUS-OpenJev 4B": "yes", |
| "APUS-OpenJev 9B": "yes", |
| "Jev official API": "yes", |
| "Laya typed / full input": "yes" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "boolq/42a6042685586d3796f9fe4460dd6e9cec208f39de59f765dcbef491fe576361", |
| "group_id": "evidence-group-23bbdb536073ab9fc895075ee72551bfc55632aaaec7a744418f9e8ada9c6993", |
| "dataset": "boolq.dev.records.jsonl", |
| "gold": "no", |
| "predictions": { |
| "APUS-OpenJev 4B": "no", |
| "APUS-OpenJev 9B": "no", |
| "Jev official API": "no", |
| "Laya typed / full input": "no" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "boolq/46e903566fe8817ff37dce61d1ef85b757aa465dadb873c6356b6fc23f58476b", |
| "group_id": "evidence-group-0598f311fcee4030bb3e055fd8b809d3932df9668305a3243086d7fdb94ea925", |
| "dataset": "boolq.dev.records.jsonl", |
| "gold": "yes", |
| "predictions": { |
| "APUS-OpenJev 4B": "yes", |
| "APUS-OpenJev 9B": "yes", |
| "Jev official API": "yes", |
| "Laya typed / full input": "yes" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "boolq/4c460ed013de1b5d82c629bbf4e3bdbdaf4441a81341404d48d8e02ead5b9814", |
| "group_id": "evidence-group-a3f2fee0d01c242524e9d213c55f9f3e9f719327de2336257b7a3aedff15bbb5", |
| "dataset": "boolq.dev.records.jsonl", |
| "gold": "yes", |
| "predictions": { |
| "APUS-OpenJev 4B": "yes", |
| "APUS-OpenJev 9B": "yes", |
| "Jev official API": "yes", |
| "Laya typed / full input": "no" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": false |
| } |
| }, |
| { |
| "record_id": "boolq/58b070021c1d992130604f309a01b95166a281de0499349346694570a6c6ac57", |
| "group_id": "evidence-group-7b4397539e19353c30f0bf4c65423110a9ac730c11d28cb2c752348d7a6e9e37", |
| "dataset": "boolq.dev.records.jsonl", |
| "gold": "no", |
| "predictions": { |
| "APUS-OpenJev 4B": "no", |
| "APUS-OpenJev 9B": "no", |
| "Jev official API": "no", |
| "Laya typed / full input": "no" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "boolq/5ba6d06215b4f7b269d0b30369fea59d748f02f41d1977f14c46c9153cd45cde", |
| "group_id": "evidence-group-0e7a5e22992089e0acef38c45a6a3764fe0cd682a8985ecfd39eb1475f5f0a1c", |
| "dataset": "boolq.dev.records.jsonl", |
| "gold": "no", |
| "predictions": { |
| "APUS-OpenJev 4B": "no", |
| "APUS-OpenJev 9B": "no", |
| "Jev official API": "no", |
| "Laya typed / full input": "no" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "boolq/6909b3bd9078683a97a30ee70c6f3e91a9b8848a34efc0784eddf12cfa305cd4", |
| "group_id": "evidence-group-ee790b93179b77e8db71445f78bd460a066b750e6f458780297e74943d7bd257", |
| "dataset": "boolq.dev.records.jsonl", |
| "gold": "no", |
| "predictions": { |
| "APUS-OpenJev 4B": "no", |
| "APUS-OpenJev 9B": "no", |
| "Jev official API": "no", |
| "Laya typed / full input": "no" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "boolq/791b5a767a61e438fe080d99068ef294b361fead41dbfc61b61021e33f2fff9f", |
| "group_id": "evidence-group-f64dca5a95a0d138d7ab52fcaffec2b80805f192bc5b892ece8256f5dfc238f7", |
| "dataset": "boolq.dev.records.jsonl", |
| "gold": "yes", |
| "predictions": { |
| "APUS-OpenJev 4B": "yes", |
| "APUS-OpenJev 9B": "yes", |
| "Jev official API": "yes", |
| "Laya typed / full input": "yes" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "boolq/8703574dc0db27742840afd3c1429829ef2f81c557670018e6ad5456fc53b150", |
| "group_id": "evidence-group-5b314cc015efa123085a0ddd9ab13485d0ac7b6809e85a039e3c43e817e1fb3e", |
| "dataset": "boolq.dev.records.jsonl", |
| "gold": "no", |
| "predictions": { |
| "APUS-OpenJev 4B": "no", |
| "APUS-OpenJev 9B": "no", |
| "Jev official API": "no", |
| "Laya typed / full input": "no" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "boolq/a1ea17311c3aea8a751a1cd170483c439ce1338b451927c7db07692878ff0d82", |
| "group_id": "evidence-group-d7abd460afbf15ef627bcaf52168fde0428e0431b694dfee9a13fa42c925b59e", |
| "dataset": "boolq.dev.records.jsonl", |
| "gold": "no", |
| "predictions": { |
| "APUS-OpenJev 4B": "no", |
| "APUS-OpenJev 9B": "no", |
| "Jev official API": "no", |
| "Laya typed / full input": "no" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "boolq/cca90e3be9b421447fe9ae0594193ab92a92adb0083f27a46bc224ad0e67c46c", |
| "group_id": "evidence-group-9735dc5df481652207fbd054ff5dad0d4ce8e7ce6c039aebe80d1e3ca8c8ee5d", |
| "dataset": "boolq.dev.records.jsonl", |
| "gold": "no", |
| "predictions": { |
| "APUS-OpenJev 4B": "no", |
| "APUS-OpenJev 9B": "no", |
| "Jev official API": "no", |
| "Laya typed / full input": "no" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "boolq/dbadfab033f2646b1a933fad5e132dc65cd8998c337d0332763f1df07680d7e7", |
| "group_id": "evidence-group-8ab83ce8fefb5340c8f2629e2ee8c09039f965976c87796f8b2d6d58b40f3017", |
| "dataset": "boolq.dev.records.jsonl", |
| "gold": "yes", |
| "predictions": { |
| "APUS-OpenJev 4B": "yes", |
| "APUS-OpenJev 9B": "yes", |
| "Jev official API": "yes", |
| "Laya typed / full input": "yes" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "goemotions:0c48f870b5edcb7a8dcfeb58087c7e3e56a9a2d89a0687e66d9974557c74556a:love", |
| "group_id": "goemotions-content:0c48f870b5edcb7a8dcfeb58087c7e3e56a9a2d89a0687e66d9974557c74556a", |
| "dataset": "score.dev.records.jsonl", |
| "gold": "no", |
| "predictions": { |
| "APUS-OpenJev 4B": "no", |
| "APUS-OpenJev 9B": "no", |
| "Jev official API": "no", |
| "Laya typed / full input": "no" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "goemotions:0d6f68c6aab93172edb30824a77a2f20e857a379845cfc16d8315af53d05bc1f:approval", |
| "group_id": "goemotions-content:0d6f68c6aab93172edb30824a77a2f20e857a379845cfc16d8315af53d05bc1f", |
| "dataset": "score.dev.records.jsonl", |
| "gold": "no", |
| "predictions": { |
| "APUS-OpenJev 4B": "yes", |
| "APUS-OpenJev 9B": "no", |
| "Jev official API": "no", |
| "Laya typed / full input": "yes" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": false, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": false |
| } |
| }, |
| { |
| "record_id": "goemotions:1285f8257d6a85acd0b9b01bfbebadff45ec085608b755eb415c74c624f8f5eb:optimism", |
| "group_id": "goemotions-content:1285f8257d6a85acd0b9b01bfbebadff45ec085608b755eb415c74c624f8f5eb", |
| "dataset": "score.dev.records.jsonl", |
| "gold": "no", |
| "predictions": { |
| "APUS-OpenJev 4B": "yes", |
| "APUS-OpenJev 9B": "yes", |
| "Jev official API": "no", |
| "Laya typed / full input": "no" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": false, |
| "APUS-OpenJev 9B": false, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "goemotions:30e748ace2950567f2e1dcc3c638f6d5fc1e2df8078792928acf9550168933c7:approval", |
| "group_id": "goemotions-content:30e748ace2950567f2e1dcc3c638f6d5fc1e2df8078792928acf9550168933c7", |
| "dataset": "score.dev.records.jsonl", |
| "gold": "no", |
| "predictions": { |
| "APUS-OpenJev 4B": "yes", |
| "APUS-OpenJev 9B": "yes", |
| "Jev official API": "yes", |
| "Laya typed / full input": "yes" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": false, |
| "APUS-OpenJev 9B": false, |
| "Jev official API": false, |
| "Laya typed / full input": false |
| } |
| }, |
| { |
| "record_id": "goemotions:3b0e3210dddfd4dbb4b0e76cf62299246fd5a912229994a9e07de7dfc64c2262:embarrassment", |
| "group_id": "goemotions-content:3b0e3210dddfd4dbb4b0e76cf62299246fd5a912229994a9e07de7dfc64c2262", |
| "dataset": "score.dev.records.jsonl", |
| "gold": "no", |
| "predictions": { |
| "APUS-OpenJev 4B": "no", |
| "APUS-OpenJev 9B": "no", |
| "Jev official API": "no", |
| "Laya typed / full input": "no" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "goemotions:4308985a124eb5964a14e35ae0d7db83da5f33c019222497141a9a455cb5500c:joy", |
| "group_id": "goemotions-content:4308985a124eb5964a14e35ae0d7db83da5f33c019222497141a9a455cb5500c", |
| "dataset": "score.dev.records.jsonl", |
| "gold": "no", |
| "predictions": { |
| "APUS-OpenJev 4B": "no", |
| "APUS-OpenJev 9B": "no", |
| "Jev official API": "no", |
| "Laya typed / full input": "no" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "goemotions:43ef087b074a832ce71850c4c8d1672ca1d68b8b03830f65be8a52da990b454c:surprise", |
| "group_id": "goemotions-content:43ef087b074a832ce71850c4c8d1672ca1d68b8b03830f65be8a52da990b454c", |
| "dataset": "score.dev.records.jsonl", |
| "gold": "no", |
| "predictions": { |
| "APUS-OpenJev 4B": "no", |
| "APUS-OpenJev 9B": "no", |
| "Jev official API": "no", |
| "Laya typed / full input": "no" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "goemotions:4c893be6bdf7ad879db4d2299867417db381418ebeed081d85cbd13e6b3371a6:anger", |
| "group_id": "goemotions-content:4c893be6bdf7ad879db4d2299867417db381418ebeed081d85cbd13e6b3371a6", |
| "dataset": "score.dev.records.jsonl", |
| "gold": "no", |
| "predictions": { |
| "APUS-OpenJev 4B": "no", |
| "APUS-OpenJev 9B": "no", |
| "Jev official API": "no", |
| "Laya typed / full input": "no" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "goemotions:5695bd603bf8ce60300298693b29b4295f2d34c083c7e52bff826db80a6ea996:grief", |
| "group_id": "goemotions-content:5695bd603bf8ce60300298693b29b4295f2d34c083c7e52bff826db80a6ea996", |
| "dataset": "score.dev.records.jsonl", |
| "gold": "no", |
| "predictions": { |
| "APUS-OpenJev 4B": "no", |
| "APUS-OpenJev 9B": "no", |
| "Jev official API": "no", |
| "Laya typed / full input": "no" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "goemotions:5bb2f0355125b6ea32d14bc489651edeb09f6b4ef9f51b90b8ed330b8fd14cef:sadness", |
| "group_id": "goemotions-content:5bb2f0355125b6ea32d14bc489651edeb09f6b4ef9f51b90b8ed330b8fd14cef", |
| "dataset": "score.dev.records.jsonl", |
| "gold": "no", |
| "predictions": { |
| "APUS-OpenJev 4B": "no", |
| "APUS-OpenJev 9B": "no", |
| "Jev official API": "no", |
| "Laya typed / full input": "no" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "goemotions:8285d89a0201a60100cf7540b9e92fbf0564e50afc4e58afb812f91b6fc64cba:desire", |
| "group_id": "goemotions-content:8285d89a0201a60100cf7540b9e92fbf0564e50afc4e58afb812f91b6fc64cba", |
| "dataset": "score.dev.records.jsonl", |
| "gold": "no", |
| "predictions": { |
| "APUS-OpenJev 4B": "no", |
| "APUS-OpenJev 9B": "no", |
| "Jev official API": "no", |
| "Laya typed / full input": "no" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "goemotions:a312dda34fe450feab6e22e6d2aa7493d77c757d4800e2bfd9904549ff4c3756:realization", |
| "group_id": "goemotions-content:a312dda34fe450feab6e22e6d2aa7493d77c757d4800e2bfd9904549ff4c3756", |
| "dataset": "score.dev.records.jsonl", |
| "gold": "no", |
| "predictions": { |
| "APUS-OpenJev 4B": "no", |
| "APUS-OpenJev 9B": "no", |
| "Jev official API": "no", |
| "Laya typed / full input": "no" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "goemotions:cfa49c3d3a46c50e494455f5f1aa94c401a83a9450181bcf5d1f64ae75e30e64:annoyance", |
| "group_id": "goemotions-content:cfa49c3d3a46c50e494455f5f1aa94c401a83a9450181bcf5d1f64ae75e30e64", |
| "dataset": "score.dev.records.jsonl", |
| "gold": "no", |
| "predictions": { |
| "APUS-OpenJev 4B": "no", |
| "APUS-OpenJev 9B": "no", |
| "Jev official API": "yes", |
| "Laya typed / full input": "no" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": false, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "goemotions:e97ef69f22521e136e92eee4bbc1e46d8571bf91280b2bc7ae24f2e1ec5d0745:confusion", |
| "group_id": "goemotions-content:e97ef69f22521e136e92eee4bbc1e46d8571bf91280b2bc7ae24f2e1ec5d0745", |
| "dataset": "score.dev.records.jsonl", |
| "gold": "no", |
| "predictions": { |
| "APUS-OpenJev 4B": "no", |
| "APUS-OpenJev 9B": "no", |
| "Jev official API": "no", |
| "Laya typed / full input": "no" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "goemotions:e97ef69f22521e136e92eee4bbc1e46d8571bf91280b2bc7ae24f2e1ec5d0745:desire", |
| "group_id": "goemotions-content:e97ef69f22521e136e92eee4bbc1e46d8571bf91280b2bc7ae24f2e1ec5d0745", |
| "dataset": "score.dev.records.jsonl", |
| "gold": "no", |
| "predictions": { |
| "APUS-OpenJev 4B": "no", |
| "APUS-OpenJev 9B": "yes", |
| "Jev official API": "no", |
| "Laya typed / full input": "no" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": false, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "goemotions:ee392e883c467546e7e603f5e6b94702211b3b4af3519a0bb7bd3b21bf4cbfd9:joy", |
| "group_id": "goemotions-content:ee392e883c467546e7e603f5e6b94702211b3b4af3519a0bb7bd3b21bf4cbfd9", |
| "dataset": "score.dev.records.jsonl", |
| "gold": "no", |
| "predictions": { |
| "APUS-OpenJev 4B": "no", |
| "APUS-OpenJev 9B": "no", |
| "Jev official API": "no", |
| "Laya typed / full input": "no" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "hs3-f6d145777bcbde96137596340fab89793acd1031-train-0000105", |
| "group_id": "hs3-context-7a2fa0f0fe7894bf1abb41d8518e6962c162827ae154f3d80b78717be95835b5", |
| "dataset": "hs3.dev.records.jsonl", |
| "gold": "yes", |
| "predictions": { |
| "APUS-OpenJev 4B": "yes", |
| "APUS-OpenJev 9B": "yes", |
| "Jev official API": "yes", |
| "Laya typed / full input": "no" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": false |
| } |
| }, |
| { |
| "record_id": "hs3-f6d145777bcbde96137596340fab89793acd1031-train-0001238", |
| "group_id": "hs3-context-6b5a52f42497fe8358c3d2ff591c263110d309fea01ff60990884ea43b9fcda2", |
| "dataset": "hs3.dev.records.jsonl", |
| "gold": "yes", |
| "predictions": { |
| "APUS-OpenJev 4B": "yes", |
| "APUS-OpenJev 9B": "yes", |
| "Jev official API": "yes", |
| "Laya typed / full input": "yes" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "hs3-f6d145777bcbde96137596340fab89793acd1031-train-0001994", |
| "group_id": "hs3-context-467315e7063b6cb4c0b2914bd11e5be08b34591c4b5b7a50b6b1cbdaacf67c5e", |
| "dataset": "hs3.dev.records.jsonl", |
| "gold": "yes", |
| "predictions": { |
| "APUS-OpenJev 4B": "no", |
| "APUS-OpenJev 9B": "no", |
| "Jev official API": "no", |
| "Laya typed / full input": "yes" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": false, |
| "APUS-OpenJev 9B": false, |
| "Jev official API": false, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "hs3-f6d145777bcbde96137596340fab89793acd1031-train-0008261", |
| "group_id": "hs3-context-0ea043fe9da180d611ed606527d5ec5d2aa4b9fd01799d01cc1a7a188592243f", |
| "dataset": "hs3.dev.records.jsonl", |
| "gold": "no", |
| "predictions": { |
| "APUS-OpenJev 4B": "no", |
| "APUS-OpenJev 9B": "no", |
| "Jev official API": "no", |
| "Laya typed / full input": "no" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "hs3-f6d145777bcbde96137596340fab89793acd1031-train-0010645", |
| "group_id": "hs3-context-e006169b0c265c71fa58ed56f685920a1b7e7e46bc2c6aeb71e68f9e72760e8b", |
| "dataset": "hs3.dev.records.jsonl", |
| "gold": "no", |
| "predictions": { |
| "APUS-OpenJev 4B": "yes", |
| "APUS-OpenJev 9B": "no", |
| "Jev official API": "yes", |
| "Laya typed / full input": "yes" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": false, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": false, |
| "Laya typed / full input": false |
| } |
| }, |
| { |
| "record_id": "hs3-f6d145777bcbde96137596340fab89793acd1031-train-0010788", |
| "group_id": "hs3-context-6c870e63acf9c7c0026b23a8adc69509642da7798f11b1d3efcd7396b04c4885", |
| "dataset": "hs3.dev.records.jsonl", |
| "gold": "no", |
| "predictions": { |
| "APUS-OpenJev 4B": "no", |
| "APUS-OpenJev 9B": "no", |
| "Jev official API": "no", |
| "Laya typed / full input": "no" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "hs3-f6d145777bcbde96137596340fab89793acd1031-train-0011197", |
| "group_id": "hs3-context-58d30fb5fab1cfda3286562df2aaa1b9b492103c730d266a26acabe8067e2254", |
| "dataset": "hs3.dev.records.jsonl", |
| "gold": "yes", |
| "predictions": { |
| "APUS-OpenJev 4B": "yes", |
| "APUS-OpenJev 9B": "yes", |
| "Jev official API": "yes", |
| "Laya typed / full input": "yes" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "hs3-f6d145777bcbde96137596340fab89793acd1031-train-0014165", |
| "group_id": "hs3-context-b086e3a56a3f29ef420f7a11867567e2464e6d4771744797ba3c9cb4f60075f2", |
| "dataset": "hs3.dev.records.jsonl", |
| "gold": "yes", |
| "predictions": { |
| "APUS-OpenJev 4B": "yes", |
| "APUS-OpenJev 9B": "yes", |
| "Jev official API": "yes", |
| "Laya typed / full input": "yes" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "hs3-f6d145777bcbde96137596340fab89793acd1031-train-0014712", |
| "group_id": "hs3-context-b7bdb036aaa37153527cb2232c2d31505b414607b3dfe4b017edafbb7ece826f", |
| "dataset": "hs3.dev.records.jsonl", |
| "gold": "yes", |
| "predictions": { |
| "APUS-OpenJev 4B": "yes", |
| "APUS-OpenJev 9B": "yes", |
| "Jev official API": "yes", |
| "Laya typed / full input": "yes" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "hs3-f6d145777bcbde96137596340fab89793acd1031-train-0015735", |
| "group_id": "hs3-context-b059df1a830f6f3f600339976f2000210e7df5eb69e99d00ebb5db5f0a0a3394", |
| "dataset": "hs3.dev.records.jsonl", |
| "gold": "no", |
| "predictions": { |
| "APUS-OpenJev 4B": "yes", |
| "APUS-OpenJev 9B": "yes", |
| "Jev official API": "yes", |
| "Laya typed / full input": "yes" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": false, |
| "APUS-OpenJev 9B": false, |
| "Jev official API": false, |
| "Laya typed / full input": false |
| } |
| }, |
| { |
| "record_id": "hs3-f6d145777bcbde96137596340fab89793acd1031-train-0017850", |
| "group_id": "hs3-context-79018d8f60c4146ca7266b4178fd5c5dd283ccfa1b2022891749ae089273259f", |
| "dataset": "hs3.dev.records.jsonl", |
| "gold": "no", |
| "predictions": { |
| "APUS-OpenJev 4B": "no", |
| "APUS-OpenJev 9B": "no", |
| "Jev official API": "no", |
| "Laya typed / full input": "yes" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": false |
| } |
| }, |
| { |
| "record_id": "hs3-f6d145777bcbde96137596340fab89793acd1031-train-0018442", |
| "group_id": "hs3-context-b6709abe80497564f903e440eb8594cc8c589156807d75b5fcde781d2d390841", |
| "dataset": "hs3.dev.records.jsonl", |
| "gold": "yes", |
| "predictions": { |
| "APUS-OpenJev 4B": "yes", |
| "APUS-OpenJev 9B": "yes", |
| "Jev official API": "no", |
| "Laya typed / full input": "yes" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": false, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "hs3-f6d145777bcbde96137596340fab89793acd1031-train-0019640", |
| "group_id": "hs3-context-dd4926767c644b6b2061bb3ef3e4662e588bd2870aa2d19e3793ca520e92da3b", |
| "dataset": "hs3.dev.records.jsonl", |
| "gold": "yes", |
| "predictions": { |
| "APUS-OpenJev 4B": "yes", |
| "APUS-OpenJev 9B": "yes", |
| "Jev official API": "yes", |
| "Laya typed / full input": "yes" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "hs3-f6d145777bcbde96137596340fab89793acd1031-train-0020714", |
| "group_id": "hs3-context-fa5f268e4c1958de8ee63a7ae4bef26c50d3d4e1d5bed178b6098657b84c20eb", |
| "dataset": "hs3.dev.records.jsonl", |
| "gold": "yes", |
| "predictions": { |
| "APUS-OpenJev 4B": "yes", |
| "APUS-OpenJev 9B": "yes", |
| "Jev official API": "yes", |
| "Laya typed / full input": "yes" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "hs3-f6d145777bcbde96137596340fab89793acd1031-train-0024669", |
| "group_id": "hs3-context-6ccc2824fad3e248830bdf04072734f1764cdf1f193fa7946ee423e4d153a860", |
| "dataset": "hs3.dev.records.jsonl", |
| "gold": "yes", |
| "predictions": { |
| "APUS-OpenJev 4B": "no", |
| "APUS-OpenJev 9B": "no", |
| "Jev official API": "yes", |
| "Laya typed / full input": "yes" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": false, |
| "APUS-OpenJev 9B": false, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "hs3-f6d145777bcbde96137596340fab89793acd1031-train-0032390", |
| "group_id": "hs3-context-cabcd8323a4a475b9f0242c93ff3c050b8a00265d87f066d65af6d1493cf3208", |
| "dataset": "hs3.dev.records.jsonl", |
| "gold": "no", |
| "predictions": { |
| "APUS-OpenJev 4B": "no", |
| "APUS-OpenJev 9B": "no", |
| "Jev official API": "no", |
| "Laya typed / full input": "yes" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": false |
| } |
| }, |
| { |
| "record_id": "mind2web:0245df99-2909-465a-861e-7fbca948e82f:dc1847f7-919b-4a2f-b778-2ee33edacc46", |
| "group_id": "mind2web-episode:0245df99-2909-465a-861e-7fbca948e82f", |
| "dataset": "browser.dev.records.jsonl", |
| "gold": "candidate-a1a3ad1a8495aa678cbe", |
| "predictions": { |
| "APUS-OpenJev 4B": "candidate-a1a3ad1a8495aa678cbe", |
| "APUS-OpenJev 9B": "candidate-a1a3ad1a8495aa678cbe", |
| "Jev official API": "candidate-a1a3ad1a8495aa678cbe", |
| "Laya typed / full input": "candidate-a1a3ad1a8495aa678cbe" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "mind2web:0b59dd33-7f6a-48df-aa1e-9cc67177287f:b99c5581-8a56-4bd7-bbe4-782795ebf93c", |
| "group_id": "mind2web-episode:0b59dd33-7f6a-48df-aa1e-9cc67177287f", |
| "dataset": "browser.dev.records.jsonl", |
| "gold": "candidate-9409b72a36db8d4a0a79", |
| "predictions": { |
| "APUS-OpenJev 4B": "candidate-9409b72a36db8d4a0a79", |
| "APUS-OpenJev 9B": "candidate-9409b72a36db8d4a0a79", |
| "Jev official API": "candidate-9409b72a36db8d4a0a79", |
| "Laya typed / full input": "candidate-9409b72a36db8d4a0a79" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "mind2web:1efdcd9d-ebc6-4bb7-8823-e54dfe25f409:f6e611c9-ad21-49ca-a841-7ad529b56c95", |
| "group_id": "mind2web-episode:1efdcd9d-ebc6-4bb7-8823-e54dfe25f409", |
| "dataset": "browser.dev.records.jsonl", |
| "gold": "candidate-fb3025ec54721504f9b5", |
| "predictions": { |
| "APUS-OpenJev 4B": "candidate-fb3025ec54721504f9b5", |
| "APUS-OpenJev 9B": "candidate-fb3025ec54721504f9b5", |
| "Jev official API": "candidate-fb3025ec54721504f9b5", |
| "Laya typed / full input": "candidate-fb3025ec54721504f9b5" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "mind2web:274571ea-fc2f-4353-86ba-00ecb112d6d2:e6345ea9-a5a4-4b88-95b5-4efececed261", |
| "group_id": "mind2web-episode:274571ea-fc2f-4353-86ba-00ecb112d6d2", |
| "dataset": "browser.dev.records.jsonl", |
| "gold": "candidate-3b25ae8d064a298ef021", |
| "predictions": { |
| "APUS-OpenJev 4B": "candidate-3b25ae8d064a298ef021", |
| "APUS-OpenJev 9B": "candidate-3b25ae8d064a298ef021", |
| "Jev official API": "candidate-3b25ae8d064a298ef021", |
| "Laya typed / full input": "candidate-9827a0d3bbbb4c3f2ef3" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": false |
| } |
| }, |
| { |
| "record_id": "mind2web:3f0988e0-e3a4-4dd7-a89f-482175a474a8:a684c98a-c238-4bef-b2ad-f476f07d73f8", |
| "group_id": "mind2web-episode:3f0988e0-e3a4-4dd7-a89f-482175a474a8", |
| "dataset": "browser.dev.records.jsonl", |
| "gold": "candidate-016db9380de74cbcab5d", |
| "predictions": { |
| "APUS-OpenJev 4B": "candidate-016db9380de74cbcab5d", |
| "APUS-OpenJev 9B": "candidate-016db9380de74cbcab5d", |
| "Jev official API": "candidate-6ff913a423628d92b636", |
| "Laya typed / full input": "candidate-ae8922f024c12c266278" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": false, |
| "Laya typed / full input": false |
| } |
| }, |
| { |
| "record_id": "mind2web:44a12ff5-0172-444a-b979-f224162c1aa8:5b8da6f5-c53c-4b69-bfad-7bdfd2e6ce20", |
| "group_id": "mind2web-episode:44a12ff5-0172-444a-b979-f224162c1aa8", |
| "dataset": "browser.dev.records.jsonl", |
| "gold": "candidate-3ef3bb624316b6eb3508", |
| "predictions": { |
| "APUS-OpenJev 4B": "candidate-94dddd06eb43ed2921b0", |
| "APUS-OpenJev 9B": "candidate-f5c1b8b97d7add30b043", |
| "Jev official API": "candidate-94dddd06eb43ed2921b0", |
| "Laya typed / full input": "candidate-331b8cbdf48fd7f61188" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": false, |
| "APUS-OpenJev 9B": false, |
| "Jev official API": false, |
| "Laya typed / full input": false |
| } |
| }, |
| { |
| "record_id": "mind2web:5b56d5b8-1f41-43ca-9f21-369d849f1aa0:edb09eb7-6a8c-4aeb-9b52-796762ca821d", |
| "group_id": "mind2web-episode:5b56d5b8-1f41-43ca-9f21-369d849f1aa0", |
| "dataset": "browser.dev.records.jsonl", |
| "gold": "candidate-2dc16f49fe3acb051b39", |
| "predictions": { |
| "APUS-OpenJev 4B": "candidate-2dc16f49fe3acb051b39", |
| "APUS-OpenJev 9B": "candidate-2dc16f49fe3acb051b39", |
| "Jev official API": "candidate-2dc16f49fe3acb051b39", |
| "Laya typed / full input": "candidate-ea45663120f48fa54b88" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": false |
| } |
| }, |
| { |
| "record_id": "mind2web:5f9182dc-d35d-4c0e-9abe-cd913c136528:8450e2c6-f8aa-40bc-876e-21cf29a8cb77", |
| "group_id": "mind2web-episode:5f9182dc-d35d-4c0e-9abe-cd913c136528", |
| "dataset": "browser.dev.records.jsonl", |
| "gold": "candidate-d2449a13158601200ec0", |
| "predictions": { |
| "APUS-OpenJev 4B": "candidate-d2449a13158601200ec0", |
| "APUS-OpenJev 9B": "candidate-d2449a13158601200ec0", |
| "Jev official API": "candidate-33ff2e219977f0035115", |
| "Laya typed / full input": "candidate-bfd1999ae1c4a3ad89f3" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": false, |
| "Laya typed / full input": false |
| } |
| }, |
| { |
| "record_id": "mind2web:6e565708-43e2-492b-9f1d-25d51387dcf7:bee75aa3-6f7d-4626-be6f-1b217ac16733", |
| "group_id": "mind2web-episode:6e565708-43e2-492b-9f1d-25d51387dcf7", |
| "dataset": "browser.dev.records.jsonl", |
| "gold": "candidate-bb1945d8d438379625ee", |
| "predictions": { |
| "APUS-OpenJev 4B": "candidate-bb1945d8d438379625ee", |
| "APUS-OpenJev 9B": "candidate-0fb9a6cde602f212d85e", |
| "Jev official API": "candidate-0fb9a6cde602f212d85e", |
| "Laya typed / full input": "candidate-69abfa80e562043e1fa3" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": false, |
| "Jev official API": false, |
| "Laya typed / full input": false |
| } |
| }, |
| { |
| "record_id": "mind2web:80e12375-19ad-400f-9e35-2a3853173bed:e89bb795-2d24-4e2c-bcae-1294e3501dfa", |
| "group_id": "mind2web-episode:80e12375-19ad-400f-9e35-2a3853173bed", |
| "dataset": "browser.dev.records.jsonl", |
| "gold": "candidate-41c5af16d7bf373cc74c", |
| "predictions": { |
| "APUS-OpenJev 4B": "candidate-41c5af16d7bf373cc74c", |
| "APUS-OpenJev 9B": "candidate-41c5af16d7bf373cc74c", |
| "Jev official API": "candidate-41c5af16d7bf373cc74c", |
| "Laya typed / full input": "candidate-41c5af16d7bf373cc74c" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "mind2web:8368b990-c6ca-4cfe-a7ab-c2a88697639d:bf14f1d4-470f-4110-b3f4-019a9f7d0aed", |
| "group_id": "mind2web-episode:8368b990-c6ca-4cfe-a7ab-c2a88697639d", |
| "dataset": "browser.dev.records.jsonl", |
| "gold": "candidate-8b6280aa9682e1a79ce0", |
| "predictions": { |
| "APUS-OpenJev 4B": "candidate-da095abe0a63c03f3114", |
| "APUS-OpenJev 9B": "candidate-e76caeae9d53c8942ed1", |
| "Jev official API": "candidate-da095abe0a63c03f3114", |
| "Laya typed / full input": "candidate-f8c1512ffc92c70eafe7" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": false, |
| "APUS-OpenJev 9B": false, |
| "Jev official API": false, |
| "Laya typed / full input": false |
| } |
| }, |
| { |
| "record_id": "mind2web:8eae88ef-9641-43c6-be6d-f8abc96d99fa:282c09d0-c9e0-4007-b88a-27887fe1e388", |
| "group_id": "mind2web-episode:8eae88ef-9641-43c6-be6d-f8abc96d99fa", |
| "dataset": "browser.dev.records.jsonl", |
| "gold": "candidate-976c2731e7518a1f1b2b", |
| "predictions": { |
| "APUS-OpenJev 4B": "candidate-976c2731e7518a1f1b2b", |
| "APUS-OpenJev 9B": "candidate-976c2731e7518a1f1b2b", |
| "Jev official API": "candidate-976c2731e7518a1f1b2b", |
| "Laya typed / full input": "candidate-7e020b58dbe771cceb6b" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": false |
| } |
| }, |
| { |
| "record_id": "mind2web:bef473f1-82a1-4359-a2c0-59b6dc2f6abb:22ad3562-e0f4-42c3-b096-8c173a47673c", |
| "group_id": "mind2web-episode:bef473f1-82a1-4359-a2c0-59b6dc2f6abb", |
| "dataset": "browser.dev.records.jsonl", |
| "gold": "candidate-c59cf6b3cdd1ee8dfe92", |
| "predictions": { |
| "APUS-OpenJev 4B": "candidate-c59cf6b3cdd1ee8dfe92", |
| "APUS-OpenJev 9B": "candidate-c59cf6b3cdd1ee8dfe92", |
| "Jev official API": "candidate-c59cf6b3cdd1ee8dfe92", |
| "Laya typed / full input": "candidate-7111d2a0c81d7ac049be" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": false |
| } |
| }, |
| { |
| "record_id": "mind2web:caafd610-202e-49d2-85d1-3f167f3ab443:d7bfb473-8c73-4808-96bc-187d00be5ad7", |
| "group_id": "mind2web-episode:caafd610-202e-49d2-85d1-3f167f3ab443", |
| "dataset": "browser.dev.records.jsonl", |
| "gold": "candidate-b56ad12ceea4c52feb3f", |
| "predictions": { |
| "APUS-OpenJev 4B": "candidate-b56ad12ceea4c52feb3f", |
| "APUS-OpenJev 9B": "candidate-b56ad12ceea4c52feb3f", |
| "Jev official API": "candidate-b56ad12ceea4c52feb3f", |
| "Laya typed / full input": "candidate-9ebb992dd5a329ac8366" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": false |
| } |
| }, |
| { |
| "record_id": "mind2web:d1de3d1a-3df1-4421-98f3-f8d078752893:e1f18ee3-1577-44fb-a283-1be215e5ae52", |
| "group_id": "mind2web-episode:d1de3d1a-3df1-4421-98f3-f8d078752893", |
| "dataset": "browser.dev.records.jsonl", |
| "gold": "candidate-888bb5997d1aba7b25f7", |
| "predictions": { |
| "APUS-OpenJev 4B": "candidate-888bb5997d1aba7b25f7", |
| "APUS-OpenJev 9B": "candidate-888bb5997d1aba7b25f7", |
| "Jev official API": "candidate-888bb5997d1aba7b25f7", |
| "Laya typed / full input": "candidate-0196e00c3795624bcc41" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": false |
| } |
| }, |
| { |
| "record_id": "mind2web:d637c171-dc6e-4a4e-a162-9c230e822932:9053f0e2-da05-4721-87b5-13edf923052b", |
| "group_id": "mind2web-episode:d637c171-dc6e-4a4e-a162-9c230e822932", |
| "dataset": "browser.dev.records.jsonl", |
| "gold": "candidate-34e89f28b090ce5327b0", |
| "predictions": { |
| "APUS-OpenJev 4B": "candidate-34e89f28b090ce5327b0", |
| "APUS-OpenJev 9B": "candidate-34e89f28b090ce5327b0", |
| "Jev official API": "candidate-34e89f28b090ce5327b0", |
| "Laya typed / full input": "candidate-d82967df7ccb99323f3d" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": false |
| } |
| }, |
| { |
| "record_id": "mnli/13f1e11494916138e91023e5bf4d2483ab5d93f00e2e1e558c5f595ca0e8e22d", |
| "group_id": "evidence-group-1ae4104ec56e749db2b0bfb640e71f29c6e29d797a0b7eefc4c77832f7de480e", |
| "dataset": "mnli.dev.records.jsonl", |
| "gold": "neutral", |
| "predictions": { |
| "APUS-OpenJev 4B": "entailment", |
| "APUS-OpenJev 9B": "entailment", |
| "Jev official API": "neutral", |
| "Laya typed / full input": "neutral" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": false, |
| "APUS-OpenJev 9B": false, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "mnli/3d8bd065ebff13c30fa3aa4943c5f2851ca8a5860bae22e25f212be97790ab28", |
| "group_id": "evidence-group-06e276f7577733f351ea270929ab121557f5ba0b4b7a7ffc8599a8dc2e7ca393", |
| "dataset": "mnli.dev.records.jsonl", |
| "gold": "entailment", |
| "predictions": { |
| "APUS-OpenJev 4B": "entailment", |
| "APUS-OpenJev 9B": "entailment", |
| "Jev official API": "entailment", |
| "Laya typed / full input": "entailment" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "mnli/4bd800d8bc83435b531bb472e9078bb054a7bb68bf31036bc12502e177dea19b", |
| "group_id": "evidence-group-fc2f08eeeca529c675ba6a68c08c292b57689624394977ddbb1c731d34a4a7e8", |
| "dataset": "mnli.dev.records.jsonl", |
| "gold": "entailment", |
| "predictions": { |
| "APUS-OpenJev 4B": "entailment", |
| "APUS-OpenJev 9B": "entailment", |
| "Jev official API": "neutral", |
| "Laya typed / full input": "entailment" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": false, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "mnli/4dc433ef7c48f1158e61aa4f1b568bac7ee2ede91001c9b8d933fd5c54e71820", |
| "group_id": "evidence-group-380981d6890572077b0d6b902aa0469166c29c7a700575399cd2fc7a8c078114", |
| "dataset": "mnli.dev.records.jsonl", |
| "gold": "entailment", |
| "predictions": { |
| "APUS-OpenJev 4B": "entailment", |
| "APUS-OpenJev 9B": "entailment", |
| "Jev official API": "entailment", |
| "Laya typed / full input": "entailment" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "mnli/533107d0478441289d303edb55e3769ea75e77ee657738576b1e86f9f523ff09", |
| "group_id": "evidence-group-7d41c94d5aacee4210e9af39c38dded413bb0ed0e66b4ab138cb0cf9acc9d175", |
| "dataset": "mnli.dev.records.jsonl", |
| "gold": "neutral", |
| "predictions": { |
| "APUS-OpenJev 4B": "contradiction", |
| "APUS-OpenJev 9B": "neutral", |
| "Jev official API": "neutral", |
| "Laya typed / full input": "contradiction" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": false, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": false |
| } |
| }, |
| { |
| "record_id": "mnli/53ac0a380403b0ccd3ae48ea7763a52664f1d1744f6244c25d5980d69fcf7f0f", |
| "group_id": "evidence-group-4743dba6976b2918897a2f0f6d686fdaccca4cb6735d4db706e4aced85eb44e2", |
| "dataset": "mnli.dev.records.jsonl", |
| "gold": "entailment", |
| "predictions": { |
| "APUS-OpenJev 4B": "entailment", |
| "APUS-OpenJev 9B": "entailment", |
| "Jev official API": "neutral", |
| "Laya typed / full input": "entailment" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": false, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "mnli/5c5d3ecddeb08dada33aaa9e323aa10bdfa371aa2f535b2cd5f4f653b0e25e9c", |
| "group_id": "evidence-group-0293cb3342154c4098832723d3775fb7f1a88b31e6126f9db882ed1a0d8b491a", |
| "dataset": "mnli.dev.records.jsonl", |
| "gold": "entailment", |
| "predictions": { |
| "APUS-OpenJev 4B": "entailment", |
| "APUS-OpenJev 9B": "entailment", |
| "Jev official API": "entailment", |
| "Laya typed / full input": "entailment" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "mnli/834677de1c970abc266e34006f2e681c7301ff5d043796d92da66364b792f235", |
| "group_id": "evidence-group-ccf7a781e6c7f6802cdc3b42bb6bad795f0ba0ae4ae9b10b286d9699ad373d3e", |
| "dataset": "mnli.dev.records.jsonl", |
| "gold": "contradiction", |
| "predictions": { |
| "APUS-OpenJev 4B": "contradiction", |
| "APUS-OpenJev 9B": "contradiction", |
| "Jev official API": "contradiction", |
| "Laya typed / full input": "contradiction" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "mnli/8f4dbf064e95ffddc782d0d50ab5e6e0fb83aab546d48b00bff5f03706183bf9", |
| "group_id": "evidence-group-cad5bd59c81306dbfbc2992418db0c5720f237fc174da87cd7360ead0af20f15", |
| "dataset": "mnli.dev.records.jsonl", |
| "gold": "neutral", |
| "predictions": { |
| "APUS-OpenJev 4B": "neutral", |
| "APUS-OpenJev 9B": "neutral", |
| "Jev official API": "neutral", |
| "Laya typed / full input": "entailment" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": false |
| } |
| }, |
| { |
| "record_id": "mnli/9cf51bc8717b59b904d198884ba5b832b53cdc96a5b1aafee7203dca9bfc824d", |
| "group_id": "evidence-group-12003df0e147aae371c07114820d8e1271ea9e77fc426c2dfcd7ea5095216b12", |
| "dataset": "mnli.dev.records.jsonl", |
| "gold": "entailment", |
| "predictions": { |
| "APUS-OpenJev 4B": "neutral", |
| "APUS-OpenJev 9B": "neutral", |
| "Jev official API": "entailment", |
| "Laya typed / full input": "entailment" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": false, |
| "APUS-OpenJev 9B": false, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "mnli/ab466125416d6bc3f307f7fd25b78e63a8d90f8f54713027fdc7d75fffd8558d", |
| "group_id": "evidence-group-1d662b8bfb238ffbb5674491c34bbdb350bd550b8d944cfdf8c1f6e3d323c69a", |
| "dataset": "mnli.dev.records.jsonl", |
| "gold": "entailment", |
| "predictions": { |
| "APUS-OpenJev 4B": "neutral", |
| "APUS-OpenJev 9B": "neutral", |
| "Jev official API": "neutral", |
| "Laya typed / full input": "entailment" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": false, |
| "APUS-OpenJev 9B": false, |
| "Jev official API": false, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "mnli/b0220f9d099b621bde205ea6efa72624cc1948cceb2502cdc38d19c3762c2ba0", |
| "group_id": "evidence-group-60d29399f85535a7eef633c31f9819b7eec6d7a9800097bf817edabf2f4314f8", |
| "dataset": "mnli.dev.records.jsonl", |
| "gold": "neutral", |
| "predictions": { |
| "APUS-OpenJev 4B": "contradiction", |
| "APUS-OpenJev 9B": "neutral", |
| "Jev official API": "neutral", |
| "Laya typed / full input": "entailment" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": false, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": false |
| } |
| }, |
| { |
| "record_id": "mnli/c8c695b22a0a846dde5f0ae3c17127f27c53f23219de2f4c97f2a9bebe1e0f81", |
| "group_id": "evidence-group-877e43849360055fc2c92bc7b5a7ee30b94048287fb4100ce4eb9004cd95610f", |
| "dataset": "mnli.dev.records.jsonl", |
| "gold": "entailment", |
| "predictions": { |
| "APUS-OpenJev 4B": "entailment", |
| "APUS-OpenJev 9B": "entailment", |
| "Jev official API": "entailment", |
| "Laya typed / full input": "entailment" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "mnli/dba1b74655afea3f9676c7a8687e80c0da196b11f2324d6e3a41c3870b9d8591", |
| "group_id": "evidence-group-c6fb07e356610f631e4b8e0884fb1cef27da866d3a1286508ad84c9a1ad50eeb", |
| "dataset": "mnli.dev.records.jsonl", |
| "gold": "entailment", |
| "predictions": { |
| "APUS-OpenJev 4B": "entailment", |
| "APUS-OpenJev 9B": "entailment", |
| "Jev official API": "entailment", |
| "Laya typed / full input": "entailment" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": true |
| } |
| }, |
| { |
| "record_id": "mnli/e6856d10a89f46cbfc7fa77bf9266c482f130090a961f5bcb10be540e7c5fc2c", |
| "group_id": "evidence-group-833d5b5523f6e081b1f65c924bed3ba030ffbfef18baba106cb0c7bce86b930c", |
| "dataset": "mnli.dev.records.jsonl", |
| "gold": "neutral", |
| "predictions": { |
| "APUS-OpenJev 4B": "neutral", |
| "APUS-OpenJev 9B": "neutral", |
| "Jev official API": "neutral", |
| "Laya typed / full input": "entailment" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": false |
| } |
| }, |
| { |
| "record_id": "mnli/e6cf83f056ba7587936d8026ff9709bc50de62e0dfb282f6b8d6aa85649e688d", |
| "group_id": "evidence-group-cd8d19e8d6333678e561021d9ffa0dbfb0b7047f3fe684fd553208ed862407a6", |
| "dataset": "mnli.dev.records.jsonl", |
| "gold": "neutral", |
| "predictions": { |
| "APUS-OpenJev 4B": "neutral", |
| "APUS-OpenJev 9B": "neutral", |
| "Jev official API": "neutral", |
| "Laya typed / full input": "entailment" |
| }, |
| "correct": { |
| "APUS-OpenJev 4B": true, |
| "APUS-OpenJev 9B": true, |
| "Jev official API": true, |
| "Laya typed / full input": false |
| } |
| } |
| ], |
| "limitations": [ |
| "Each task has only 16 questions; one answer changes task accuracy by 6.25 percentage points.", |
| "Development panel was reused during development and model selection; not evidence of general superiority.", |
| "Browser measures static candidate selection, not live website task completion.", |
| "Score includes no positive labels; an always-No predictor scores 16/16.", |
| "Laya uses the typed configuration with full-input formatting; three process repeats do not increase the independent sample size." |
| ] |
| } |
|
|
