File size: 24,757 Bytes
28cf730 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 | {
"schema_version": 1,
"configuration": {
"version": "community-hard-mix-v2",
"seed": 20260921,
"replay_cap_per_source": 1500,
"input_bindings": [
{
"category": "new",
"path": "data/community-diversity-v2",
"manifest_sha256": "90a1f02ea457e65d3e3c40859fb73c5113e090fcc7dfe3cead72dbe6730bb2e1",
"files_sha256": {
"train": "4274569425c7003a8705f804b27402d6e0963167c7a21dd8c491573b40520f16",
"calibration": "ed07c44e9d2793f5e4f3449e48152ffe3915a7fed89edac16f4cd7861ccb3cd7",
"validation": "b12851e59eb3eee5d917baddde35ab7bfc89910f77d91f4a554937ec59edf833",
"test": "71168841444f9d478e6bbca6148a46a4dc4426f12f283205d6a02517145f7a6d",
"ood": "4359ec2a43d0d17d7b4f8059954af648431cff16eb4cfc97c27894434c26c49a"
}
},
{
"category": "new",
"path": "data/wanli-decisions-v1",
"manifest_sha256": "38d50b03e62c63c561eff879351a73c426fec3a431b8cedd5edb0d564aa0f5af",
"files_sha256": {
"train": "62b2561f1a43d157eb854916a4d24047f78e61dddb2e700b49d3268111449a85",
"calibration": "7253def87adf070728e534f39284df1e349e30a13c1d65291d5189afc7f0aa70",
"validation": "601ced227bb054b035654073c74eb4bafef29b3e368c325b083dab5c24fae384",
"test": "5383a944978701468b8bdc884ed7ad41e78f2aea369317feb08b931af865de4f",
"ood": "ce1fbebd84b6ca95d3dd94b2e93b224412a78090d9cafc2d4d560e61690aa13e"
}
},
{
"category": "replay",
"path": "data/browser-drone-expansion-v1",
"manifest_sha256": "ba001b88787ce21576017897a0cbdedd5917ceb29ad3522b0e1fb7b4836d49df",
"files_sha256": {
"train": "91a3e3c715abd735509a5a23453161475aff85626f2c47563a8fab4e7dc79239",
"calibration": "8a366dbb71dbe57b98a25da3071fc858beb7e8567adf4abf952c3d98f79bb94c",
"validation": "bb49a3ca608dbeb543c56a20ee2f32e75f65be835fcb40ec40014af7351c6c7d",
"test": "662d5e79da70eba77ad740eb37576b8157e53e5c1d8cd9344c60ec56d3969637",
"ood": "364992ae82a2d8b6f3f4ed68a0efa06d9a6b9ae469d81113d4165f762eec0a94"
}
},
{
"category": "replay",
"path": "data/ir-control-v1",
"manifest_sha256": "5305dad7326fbf32aa006447137c5a3eb92872d90cde71302bc7efb39f882590",
"files_sha256": {
"train": "6d33876576fb7989abbf9edcd878074f4da7d4f502d770d110a848bbe259541d",
"calibration": "7ad15592ec8b1d2e6c2d819fb186dc2dec47f8ed9214072f6f45452be7194e61",
"validation": "0ecbe867ae894a83101805328be1b1d983cc210fb624fa78d2aedd59f8b702ec",
"test": "97637e12062c98447b49cfa8cc9669e86525dce0078657a08fd9f1b54e5e2869",
"ood": "fa6332b13395ef37ab4c88e4c5888201f8c814fdd26d387bc5684e7bf410076b"
}
},
{
"category": "replay",
"path": "data/mailroom-control-v1",
"manifest_sha256": "afa9daabd5639a333c9adf8e89e7c7bbfcf84adfc051fb9843978c29703122f6",
"files_sha256": {
"train": "5a19ebf41125093c8bb7aa0861842673686014b2c288e79bfbb54a0ea1a4a86d",
"calibration": "9765f7481ff6eda49733d1cdee648c771bb1a2055bcc7df9bac5d2acda9b592b",
"validation": "97558afbd86cb84cada925c9513fa868db3e47db83e51fc1dafe955240d05516",
"test": "9bb9976cfa9e3e5dc084b86d9cbeb0bc201465a5ca0249af6781c02bdf635938",
"ood": "5667325039486c28ade0b752c3fbf694aa7c73e7998a274744b09b5dea207d0e"
}
},
{
"category": "replay",
"path": "data/amount-extraction-control-v1",
"manifest_sha256": "6f89f81336ac36505c563193da27fca2155fa566175451854f0ac410979ea80b",
"files_sha256": {
"train": "68a34ca1371b60dc9b49c7d26882893758c16bc1dbb85a636cf0643ed7f5047d",
"calibration": "4db65ea37fc9569f3666d87762aec37c6786dfa7e7caee9dc70ef6e1420d6e6b",
"validation": "b9bc54dd4c39eb1ab40313bf0ce1339c48b579077000b5de1c0a9ffaba802e76",
"test": "f8500ef9dbd65280b2c58edcfc579dd175ddde52bc2b19b6b97c5cd84bff8bd8",
"ood": "11f3348654087a561fc3ec934144712e38decf6cfb43c21efb5b96099e0f3e93"
}
},
{
"category": "replay",
"path": "data/entity-alignment-control-v1",
"manifest_sha256": "b3afddbeb926d57b7d7ea13c5fbda0fff878c647ea9ff1ee8a09b11d7f2f9eb5",
"files_sha256": {
"train": "ff4f67f3425ce6568388c44524f04fce438c580d5eb3f4bb0a8f5508f11e4234",
"calibration": "dd6abd73a852a6067bca90bb9f378b0eae8acb5cfcaf8df71473c8f64494c01f",
"validation": "e0a3d143d6dc4a0e983124746bb02537ab200a322aba84a176daea499c75e2b6",
"test": "ae101aa7d0692106f71634ccebf39b7062cb37181025af9b9b8955b0a82a0048",
"ood": "8c4172dc683c41caf7355291b1bd9e27aa558b7073037bde9a87dd3a5cc56359"
}
},
{
"category": "replay",
"path": "data/email-selection-control-v1",
"manifest_sha256": "4c18cc791425003c5c8f052ce27ed034dfb54c79ddac24de13f7290ab3b3316c",
"files_sha256": {
"train": "2127d52a792e1e7d2727315aaf7368e61b50d9853559f4e694aa96964e3fe492",
"calibration": "d6a6ddb37d4d086028a8c68f6b2c677ff4860ac0441c35e586d4f2233aa4bdd9",
"validation": "575c2a53df3b075210cb6f4845e94a15123321af0945208b0b7eba00cb044ad9",
"test": "9d4138f70de5473700dd59a92bb893d87e68d98ce0a76ead98c907fe8256e28e",
"ood": "975ec0cff2afd7188313682b73de64be76ad7f8fdc42603fefb1c2cc616ecd04"
}
},
{
"category": "replay",
"path": "data/phone-extraction-control-v1",
"manifest_sha256": "1001f1e04795308bf4f68db51076e8e1e1e47802986de54fc33b3b8d33548cfb",
"files_sha256": {
"train": "407bd4c4d0b9e703d83e2db6327acb2604f032188a0535578bce7d9d4688859e",
"calibration": "48a9d3aa1da99f6a8794e5a139e9b3d3a2b6f127edfab478e2f00d2d6c119dca",
"validation": "7a078bb9b2bd0b4c58858ace347ad4a9c513122b628799193c1214b5ff6ff395",
"test": "dfc1180917d4c8717bc15c5c2162e2dbea24172158d593b062c8cb5eb1360903",
"ood": "576fb66309089809a4646aeb88e3cc3d187c42498eb6d564c9bc79cad730f88b"
}
},
{
"category": "replay",
"path": "data/citation-control-v1",
"manifest_sha256": "069466d7536c579812bf2c870a9ea209d13a695005c299debffe61cd3aeffff4",
"files_sha256": {
"train": "c8e939ea7e49290a8d551b93cf759dda68dd25bce264dbd0461cf2fb1e7ca448",
"calibration": "feae8a0adb34690d0dadb920213b3da3f29b52aaf4c938bd3279c58821a69e04",
"validation": "e3b7dd9759199d9aa64e1df94b1efdd0ef91a7fdc7090ce0aa6943604fefb1f7",
"test": "fd55095c6387a3027e0eb9639e3c6a802cdff29e587ef3118457d84285016a50",
"ood": "f85863a39afaf12cb5ab3e4ab554f9bba93a41b1741629daa6a855c9dd3520eb"
}
},
{
"category": "replay",
"path": "data/sponsor-segment-control-v1",
"manifest_sha256": "6a143ede0485b22588fc92f4612f41428c46b1fa7d8f8cc0d32422d09b548f13",
"files_sha256": {
"train": "cd8a28b09840594f945d389447e77f0d216c852028d4cce05cda28f2506f7148",
"calibration": "79540596a197fc4364fb3fe4c7525ad8347ee72e48fec3cbfa0b43e5b47b3322",
"validation": "12a503bb310414659469ddf14aede84b5a64f4dc166bcbf28d870b9fb3a01c60",
"test": "642f6e6577a4761dee2b862f914978bd6fc6697979e1a954fd9eb9e163865a17",
"ood": "29ad7566e7b4989c7cec7aee6d8266db26d19a0829d43a5e179cdd2fade325d9"
}
},
{
"category": "replay",
"path": "data/silent-failure-control-v1",
"manifest_sha256": "a7d326a480c0132dd608d0d7119a948bc16e36667e0ff7049e111f33c5445fac",
"files_sha256": {
"train": "3b83b1d937ca2df42e94d15a7ef70125672f653ceeb82fe3790a7b07d2bf8a8f",
"calibration": "2c86b1ddc82f20fe6979920620caa5dbb3568dcc159c307ee32b49e99bc2c85d",
"validation": "173e5a00b1a5573d91ea5ba323bea2b1d4358eea44cceee85dce5823d9ea221a",
"test": "37c95060bd0c38d9dd3315e30d60ad69a38d8fe717fca32e7c77b4ed4a944024",
"ood": "3fd69c6e86ab7e6b6034e4b48bd55d845488efca1815626af9b9103e30cfd299"
}
},
{
"category": "replay",
"path": "data/context-retention-control-v1",
"manifest_sha256": "03badf69899809919e90bd299c5ddea4b3f8d646505d02f9544a1312e13bbd94",
"files_sha256": {
"train": "d6c4cdc5572da328675b9e3c5eb8af8ef186a9a3fbd5c3bdeaec4a7be21bca06",
"calibration": "b157cd03922bd89056ca3678223c1f64bd152d81d5ba4c14b1ec0088f4f4d2ca",
"validation": "20063011a0fbdc07582ce8da7e7f40502995d03460e73d7ac902a23bb32bc7cf",
"test": "a5e3adc7dbb90209ef4ffd657bcb004424ab71807bdf076ae234626cb769b3ee",
"ood": "558db34ab2be4c12fffa611a47612506977245e2d4c598caa561c374c7f68b97"
}
}
],
"replay_policy": "Hash-ordered round robin across parent groups; no duplication or oversampling.",
"new_source_policy": "One full pass over retained rows; four-rank training wraps at most three rows to complete its final step.",
"excluded_groups": [],
"filtered": {
"replay_train_cap": 232619
}
},
"summary": {
"records": 328672,
"groups": 76619,
"unique_inputs": 327954,
"splits": {
"calibration": 26764,
"ood": 84486,
"test": 43301,
"train": 148639,
"validation": 25482
},
"kinds": {
"choice": 185660,
"noul": 114562,
"score": 28450
},
"families": {
"doom_basic": 4239,
"evidence": 112873,
"natural_language_inference": 101207,
"policy": 72344,
"routing": 17720,
"rubric": 18236,
"wikiracing": 2053
},
"sources": {
"amount-extraction-control-v1": 18116,
"browser-control-v1": 9666,
"citation-control-v1": 2980,
"community-diversity-v2/browser_tools/argument_binding": 640,
"community-diversity-v2/browser_tools/confirmation_gate": 640,
"community-diversity-v2/browser_tools/date_range_form": 640,
"community-diversity-v2/browser_tools/goal_completion": 640,
"community-diversity-v2/browser_tools/idempotent_retry": 640,
"community-diversity-v2/browser_tools/pagination": 640,
"community-diversity-v2/browser_tools/stale_dom": 640,
"community-diversity-v2/browser_tools/tool_scope": 640,
"community-diversity-v2/browser_tools/typing_fallback": 640,
"community-diversity-v2/browser_tools/upload_constraint": 640,
"community-diversity-v2/contracts/assignment_consent": 640,
"community-diversity-v2/contracts/audit_right": 640,
"community-diversity-v2/contracts/automatic_renewal": 640,
"community-diversity-v2/contracts/change_of_control": 640,
"community-diversity-v2/contracts/confidentiality_survival": 640,
"community-diversity-v2/contracts/exclusivity_exception": 640,
"community-diversity-v2/contracts/governing_law_venue": 640,
"community-diversity-v2/contracts/liability_cap": 640,
"community-diversity-v2/contracts/noncompete_scope": 640,
"community-diversity-v2/contracts/termination_cure": 640,
"community-diversity-v2/games/crafting_dependencies": 640,
"community-diversity-v2/games/escort_sync": 640,
"community-diversity-v2/games/health_pickup": 640,
"community-diversity-v2/games/pit_stop": 640,
"community-diversity-v2/games/projectile_dodge": 640,
"community-diversity-v2/games/racing_braking": 640,
"community-diversity-v2/games/racing_overtake": 640,
"community-diversity-v2/games/reload_cover": 640,
"community-diversity-v2/games/safe_path": 640,
"community-diversity-v2/games/weapon_selection": 640,
"community-diversity-v2/ir/access_schedule": 640,
"community-diversity-v2/ir/api_auth_migration": 640,
"community-diversity-v2/ir/connecting_route": 640,
"community-diversity-v2/ir/hardware_manual": 640,
"community-diversity-v2/ir/incident_runbook": 640,
"community-diversity-v2/ir/inventory_answer": 640,
"community-diversity-v2/ir/regional_returns": 640,
"community-diversity-v2/ir/release_migration": 640,
"community-diversity-v2/ir/study_population": 640,
"community-diversity-v2/ir/warranty_transfer": 640,
"community-diversity-v2/rag/citation_coverage": 640,
"community-diversity-v2/rag/contradiction_resolution": 640,
"community-diversity-v2/rag/entity_alias_chain": 640,
"community-diversity-v2/rag/independent_sources": 640,
"community-diversity-v2/rag/negative_evidence": 640,
"community-diversity-v2/rag/numerical_claim": 640,
"community-diversity-v2/rag/policy_exception": 640,
"community-diversity-v2/rag/qualified_comparison": 640,
"community-diversity-v2/rag/temporal_fact": 640,
"community-diversity-v2/rag/two_hop_ownership": 640,
"community-diversity-v2/rubric_judge/abstention_quality": 640,
"community-diversity-v2/rubric_judge/balanced_tradeoff": 640,
"community-diversity-v2/rubric_judge/citation_attribution": 640,
"community-diversity-v2/rubric_judge/comparison_fairness": 640,
"community-diversity-v2/rubric_judge/faithful_summary": 640,
"community-diversity-v2/rubric_judge/format_and_content": 640,
"community-diversity-v2/rubric_judge/instruction_priority": 640,
"community-diversity-v2/rubric_judge/negation_preservation": 640,
"community-diversity-v2/rubric_judge/trace_consistency": 640,
"community-diversity-v2/rubric_judge/unit_conversion": 640,
"community-diversity-v2/shell_history/archive_extract": 640,
"community-diversity-v2/shell_history/checksum_verify": 640,
"community-diversity-v2/shell_history/dependency_install": 640,
"community-diversity-v2/shell_history/exact_repo_test": 640,
"community-diversity-v2/shell_history/git_history_query": 640,
"community-diversity-v2/shell_history/incremental_copy": 640,
"community-diversity-v2/shell_history/log_time_window": 640,
"community-diversity-v2/shell_history/port_inspection": 640,
"community-diversity-v2/shell_history/quoted_filename": 640,
"community-diversity-v2/shell_history/search_scope": 640,
"community-diversity-v2/support/account_recovery": 640,
"community-diversity-v2/support/damaged_order": 640,
"community-diversity-v2/support/delayed_parcel": 640,
"community-diversity-v2/support/duplicate_charge": 640,
"community-diversity-v2/support/outage_compensation": 640,
"community-diversity-v2/support/partial_return": 640,
"community-diversity-v2/support/privacy_export": 640,
"community-diversity-v2/support/refund_window": 640,
"community-diversity-v2/support/subscription_cancellation": 640,
"community-diversity-v2/support/warranty_repair": 640,
"context-retention-control-v1": 5196,
"customer-control-v1": 3294,
"drone-control-v1": 11055,
"email-selection-control-v1": 3282,
"entity-alignment-control-v1": 5756,
"ir-control-v1": 5734,
"mailroom-control-v1": 42828,
"painting-geometry-v1": 12508,
"phone-extraction-control-v1": 8150,
"reasoning-control-v1": 3558,
"silent-failure-control-v1": 4668,
"snake-v1": 7876,
"sponsor-segment-control-v1": 5955,
"tic_tac_toe-v1": 2534,
"tile_platformer-v1": 1697,
"trex_runner-v1": 36,
"vizdoom-basic-v1": 4239,
"wanli-decisions-v1": 101207,
"wikispeedia-v1": 2053,
"workflow-controls-v1/agent_trace_observability": 2970,
"workflow-controls-v1/customer_service": 3108,
"workflow-controls-v1/invoice_processing": 3630,
"workflow-controls-v1/security_incidents": 5376
}
},
"files_sha256": {
"train.jsonl": "959dac64adc1717d92e9bb80371d4b940b2b5994f9e36e469b3eaed85883fdff",
"calibration.jsonl": "6a02712bc6dce4c99cae3acfa54cf049553d12e4b795cfa4c170bd7de5732981",
"validation.jsonl": "743ec57c076fee300bfe3e3eb364a8fb629addfd185724c0b66d58633bcd9372",
"test.jsonl": "2cd250906060a570e0c596f7bb6ee7fb299fbf314e011a8c72ecb78d70248295",
"ood.jsonl": "b575698e0f1006713ca736b96d1d8eb43a54d052e266c0f363b027b288f54498"
},
"model_input_fields": [
"state",
"question",
"kind",
"options"
],
"counts": {
"train": 148639,
"calibration": 26764,
"validation": 25482,
"test": 43301,
"ood": 84486
},
"sha256": {
"train": "959dac64adc1717d92e9bb80371d4b940b2b5994f9e36e469b3eaed85883fdff",
"calibration": "6a02712bc6dce4c99cae3acfa54cf049553d12e4b795cfa4c170bd7de5732981",
"validation": "743ec57c076fee300bfe3e3eb364a8fb629addfd185724c0b66d58633bcd9372",
"test": "2cd250906060a570e0c596f7bb6ee7fb299fbf314e011a8c72ecb78d70248295",
"ood": "b575698e0f1006713ca736b96d1d8eb43a54d052e266c0f363b027b288f54498"
},
"training_by_source": {
"amount-extraction-control-v1": 1500,
"browser-control-v1": 1500,
"citation-control-v1": 1500,
"community-diversity-v2/browser_tools/date_range_form": 640,
"community-diversity-v2/browser_tools/pagination": 640,
"community-diversity-v2/browser_tools/stale_dom": 640,
"community-diversity-v2/browser_tools/tool_scope": 640,
"community-diversity-v2/browser_tools/typing_fallback": 640,
"community-diversity-v2/browser_tools/upload_constraint": 640,
"community-diversity-v2/contracts/assignment_consent": 640,
"community-diversity-v2/contracts/audit_right": 640,
"community-diversity-v2/contracts/automatic_renewal": 640,
"community-diversity-v2/contracts/confidentiality_survival": 640,
"community-diversity-v2/contracts/liability_cap": 640,
"community-diversity-v2/contracts/noncompete_scope": 640,
"community-diversity-v2/games/escort_sync": 640,
"community-diversity-v2/games/health_pickup": 640,
"community-diversity-v2/games/projectile_dodge": 640,
"community-diversity-v2/games/reload_cover": 640,
"community-diversity-v2/games/safe_path": 640,
"community-diversity-v2/games/weapon_selection": 640,
"community-diversity-v2/ir/connecting_route": 640,
"community-diversity-v2/ir/hardware_manual": 640,
"community-diversity-v2/ir/inventory_answer": 640,
"community-diversity-v2/ir/regional_returns": 640,
"community-diversity-v2/ir/release_migration": 640,
"community-diversity-v2/ir/study_population": 640,
"community-diversity-v2/rag/citation_coverage": 640,
"community-diversity-v2/rag/contradiction_resolution": 640,
"community-diversity-v2/rag/negative_evidence": 640,
"community-diversity-v2/rag/numerical_claim": 640,
"community-diversity-v2/rag/temporal_fact": 640,
"community-diversity-v2/rag/two_hop_ownership": 640,
"community-diversity-v2/rubric_judge/abstention_quality": 640,
"community-diversity-v2/rubric_judge/comparison_fairness": 640,
"community-diversity-v2/rubric_judge/faithful_summary": 640,
"community-diversity-v2/rubric_judge/format_and_content": 640,
"community-diversity-v2/rubric_judge/negation_preservation": 640,
"community-diversity-v2/rubric_judge/unit_conversion": 640,
"community-diversity-v2/shell_history/archive_extract": 640,
"community-diversity-v2/shell_history/dependency_install": 640,
"community-diversity-v2/shell_history/exact_repo_test": 640,
"community-diversity-v2/shell_history/git_history_query": 640,
"community-diversity-v2/shell_history/quoted_filename": 640,
"community-diversity-v2/shell_history/search_scope": 640,
"community-diversity-v2/support/account_recovery": 640,
"community-diversity-v2/support/delayed_parcel": 640,
"community-diversity-v2/support/duplicate_charge": 640,
"community-diversity-v2/support/refund_window": 640,
"community-diversity-v2/support/subscription_cancellation": 640,
"community-diversity-v2/support/warranty_repair": 640,
"context-retention-control-v1": 1500,
"customer-control-v1": 1500,
"drone-control-v1": 1500,
"email-selection-control-v1": 1500,
"entity-alignment-control-v1": 1500,
"ir-control-v1": 1500,
"mailroom-control-v1": 1500,
"painting-geometry-v1": 1500,
"phone-extraction-control-v1": 1500,
"reasoning-control-v1": 1500,
"silent-failure-control-v1": 1500,
"snake-v1": 1500,
"sponsor-segment-control-v1": 1500,
"tic_tac_toe-v1": 1500,
"tile_platformer-v1": 1346,
"trex_runner-v1": 28,
"vizdoom-basic-v1": 1500,
"wanli-decisions-v1": 82045,
"wikispeedia-v1": 1500,
"workflow-controls-v1/agent_trace_observability": 1500,
"workflow-controls-v1/customer_service": 1500,
"workflow-controls-v1/invoice_processing": 1500,
"workflow-controls-v1/security_incidents": 1500
},
"training_groups_by_source": {
"amount-extraction-control-v1": 133,
"browser-control-v1": 631,
"citation-control-v1": 126,
"community-diversity-v2/browser_tools/date_range_form": 80,
"community-diversity-v2/browser_tools/pagination": 80,
"community-diversity-v2/browser_tools/stale_dom": 80,
"community-diversity-v2/browser_tools/tool_scope": 80,
"community-diversity-v2/browser_tools/typing_fallback": 80,
"community-diversity-v2/browser_tools/upload_constraint": 80,
"community-diversity-v2/contracts/assignment_consent": 80,
"community-diversity-v2/contracts/audit_right": 80,
"community-diversity-v2/contracts/automatic_renewal": 80,
"community-diversity-v2/contracts/confidentiality_survival": 80,
"community-diversity-v2/contracts/liability_cap": 80,
"community-diversity-v2/contracts/noncompete_scope": 80,
"community-diversity-v2/games/escort_sync": 80,
"community-diversity-v2/games/health_pickup": 80,
"community-diversity-v2/games/projectile_dodge": 80,
"community-diversity-v2/games/reload_cover": 80,
"community-diversity-v2/games/safe_path": 80,
"community-diversity-v2/games/weapon_selection": 80,
"community-diversity-v2/ir/connecting_route": 80,
"community-diversity-v2/ir/hardware_manual": 80,
"community-diversity-v2/ir/inventory_answer": 80,
"community-diversity-v2/ir/regional_returns": 80,
"community-diversity-v2/ir/release_migration": 80,
"community-diversity-v2/ir/study_population": 80,
"community-diversity-v2/rag/citation_coverage": 80,
"community-diversity-v2/rag/contradiction_resolution": 80,
"community-diversity-v2/rag/negative_evidence": 80,
"community-diversity-v2/rag/numerical_claim": 80,
"community-diversity-v2/rag/temporal_fact": 80,
"community-diversity-v2/rag/two_hop_ownership": 80,
"community-diversity-v2/rubric_judge/abstention_quality": 80,
"community-diversity-v2/rubric_judge/comparison_fairness": 80,
"community-diversity-v2/rubric_judge/faithful_summary": 80,
"community-diversity-v2/rubric_judge/format_and_content": 80,
"community-diversity-v2/rubric_judge/negation_preservation": 80,
"community-diversity-v2/rubric_judge/unit_conversion": 80,
"community-diversity-v2/shell_history/archive_extract": 80,
"community-diversity-v2/shell_history/dependency_install": 80,
"community-diversity-v2/shell_history/exact_repo_test": 80,
"community-diversity-v2/shell_history/git_history_query": 80,
"community-diversity-v2/shell_history/quoted_filename": 80,
"community-diversity-v2/shell_history/search_scope": 80,
"community-diversity-v2/support/account_recovery": 80,
"community-diversity-v2/support/delayed_parcel": 80,
"community-diversity-v2/support/duplicate_charge": 80,
"community-diversity-v2/support/refund_window": 80,
"community-diversity-v2/support/subscription_cancellation": 80,
"community-diversity-v2/support/warranty_repair": 80,
"context-retention-control-v1": 255,
"customer-control-v1": 701,
"drone-control-v1": 311,
"email-selection-control-v1": 134,
"entity-alignment-control-v1": 124,
"ir-control-v1": 127,
"mailroom-control-v1": 256,
"painting-geometry-v1": 46,
"phone-extraction-control-v1": 130,
"reasoning-control-v1": 1500,
"silent-failure-control-v1": 268,
"snake-v1": 1500,
"sponsor-segment-control-v1": 235,
"tic_tac_toe-v1": 452,
"tile_platformer-v1": 63,
"trex_runner-v1": 28,
"vizdoom-basic-v1": 309,
"wanli-decisions-v1": 44541,
"wikispeedia-v1": 224,
"workflow-controls-v1/agent_trace_observability": 180,
"workflow-controls-v1/customer_service": 183,
"workflow-controls-v1/invoice_processing": 179,
"workflow-controls-v1/security_incidents": 174
},
"full_pass_four_gpu_steps": 37160,
"full_pass_four_gpu_rows_consumed": 148640,
"full_pass_four_gpu_wrapped_rows": 1
}
|