| { |
| "by_category": { |
| "basic": { |
| "check_pass": 30, |
| "check_pass_contract_clean": 30, |
| "contract_clean": 30, |
| "missing_predictions": 0, |
| "rows": 31 |
| }, |
| "effects_algebra": { |
| "check_pass": 29, |
| "check_pass_contract_clean": 29, |
| "contract_clean": 30, |
| "missing_predictions": 0, |
| "rows": 30 |
| }, |
| "ffi_interop": { |
| "check_pass": 31, |
| "check_pass_contract_clean": 31, |
| "contract_clean": 31, |
| "missing_predictions": 0, |
| "rows": 31 |
| }, |
| "gpu_octonion": { |
| "check_pass": 29, |
| "check_pass_contract_clean": 29, |
| "contract_clean": 30, |
| "missing_predictions": 0, |
| "rows": 30 |
| }, |
| "humaneval_mbpp": { |
| "check_pass": 16, |
| "check_pass_contract_clean": 16, |
| "contract_clean": 17, |
| "missing_predictions": 0, |
| "rows": 17 |
| }, |
| "knowledge": { |
| "check_pass": 31, |
| "check_pass_contract_clean": 31, |
| "contract_clean": 31, |
| "missing_predictions": 0, |
| "rows": 31 |
| }, |
| "reserve_advanced": { |
| "check_pass": 26, |
| "check_pass_contract_clean": 26, |
| "contract_clean": 27, |
| "missing_predictions": 0, |
| "rows": 30 |
| } |
| }, |
| "check_pass": 192, |
| "check_pass_contract_clean": 192, |
| "check_pass_contract_clean_rate_on_predictions": 0.96, |
| "check_rate": 0.96, |
| "check_rate_on_predictions": 0.96, |
| "contract_clean": 196, |
| "contract_clean_rate_on_predictions": 0.98, |
| "contract_violation_counts": { |
| "rust-python:.to_string()": 2, |
| "rust-python:String": 3, |
| "rust-python:Vec<": 1, |
| "rust-python:vec!": 1 |
| }, |
| "missing_predictions": 0, |
| "mode": "predictions", |
| "predictions_found": 200, |
| "results": [ |
| { |
| "category": "basic", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0002_basic_algebra_properties_basic", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "basic", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0007_basic_effect_handler_basic", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "basic", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0012_basic_import_basic_a", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "basic", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0017_basic_native_match_int", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "basic", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0022_basic_unobserved_basic", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "humaneval_mbpp", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "expert", |
| "has_prediction": true, |
| "id": "0030_humaneval_mbpp_004_palindrome", |
| "run_output_tail": "004_palindrome: ALL TESTS PASSED\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "humaneval_mbpp", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0035_humaneval_mbpp_009_power", |
| "run_output_tail": "009_power: ALL TESTS PASSED\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "humaneval_mbpp", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "expert", |
| "has_prediction": true, |
| "id": "0040_humaneval_mbpp_014_rotate_array", |
| "run_output_tail": "014_rotate_array: ALL TESTS PASSED\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "humaneval_mbpp", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0045_humaneval_mbpp_019_string_to_int", |
| "run_output_tail": "019_string_to_int: ALL TESTS PASSED\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "humaneval_mbpp", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "expert", |
| "has_prediction": true, |
| "id": "0050_humaneval_mbpp_024_histogram", |
| "run_output_tail": "024_histogram: ALL TESTS PASSED\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "humaneval_mbpp", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0055_humaneval_mbpp_029_selection_sort", |
| "run_output_tail": "029_selection_sort: ALL TESTS PASSED\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "humaneval_mbpp", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "expert", |
| "has_prediction": true, |
| "id": "0060_humaneval_mbpp_034_majority_element", |
| "run_output_tail": "034_majority_element: ALL TESTS PASSED\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "humaneval_mbpp", |
| "check_pass": false, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "source: /tmp/sounio-eval-_4fw7nrx/0065_humaneval_mbpp_039_house_robber.sio 412\n bytes\ntotal source: 10508\n bytes\ntokens: 2510\n\nst_count: 0\n depth_end: 0\n\nglobals: 0\n bss=1048864\n\neffects: scanned 23\n functions\nrefined: 54\n in 2\n passes\nunc_detail: dot=0\n decl=0\n let=0\n colon=0\n other=0\n\nepistemic: 1451\n expr, 1451\n certain (100\n%), 0\n uncertain [ident=25\n ops=29\n other=0\n], types=7\n, gate=950\n/1000\nepistemic_main: 55\n expr, 55\n certain (100\n%), 0\n uncertain\nknightian: 0\n sites, gap_sum=0\n\nknowledge_subtype: 0\n sites, 0\n violations\nknowledge_units: 0\n sites with dimensional Knowledge<T>\ntier_dist: PLATINUM=1451\n GOLD=0\n SILVER=0\n BRONZE=0\n [PLATINUM%=100\n%]\neconf: total=1451\n platinum=1451\n gate_pass=1451\n knightian=0\n min=1000\n mean=1000\n gate=950\n\nerror: unknown identifier at line 5\n\ntypecheck: failed\n", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0065_humaneval_mbpp_039_house_robber", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": true |
| }, |
| { |
| "category": "humaneval_mbpp", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "expert", |
| "has_prediction": true, |
| "id": "0070_humaneval_mbpp_044_decode_ways", |
| "run_output_tail": "044_decode_ways: ALL TESTS PASSED\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "humaneval_mbpp", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0075_humaneval_mbpp_049_count_inversions", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "humaneval_mbpp", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "expert", |
| "has_prediction": true, |
| "id": "0080_humaneval_mbpp_054_valid_bst", |
| "run_output_tail": "054_valid_bst: ALL TESTS PASSED\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "humaneval_mbpp", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0085_humaneval_mbpp_060_minimum_window", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "humaneval_mbpp", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "expert", |
| "has_prediction": true, |
| "id": "0090_humaneval_mbpp_066_surrounded_regions", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "humaneval_mbpp", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0095_humaneval_mbpp_073_rabin_karp", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "humaneval_mbpp", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "expert", |
| "has_prediction": true, |
| "id": "0100_humaneval_mbpp_080_search_rotated", |
| "run_output_tail": "080_search_rotated: ALL TESTS PASSED\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "humaneval_mbpp", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0105_humaneval_mbpp_085_add_binary", |
| "run_output_tail": "0105_humaneval_mbpp_085_add_binary: ALL TESTS PASSED\n", |
| "run_pass": false, |
| "runs_expected": true |
| }, |
| { |
| "category": "humaneval_mbpp", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "expert", |
| "has_prediction": true, |
| "id": "0110_humaneval_mbpp_092_popcount", |
| "run_output_tail": "092_popcount: ALL TESTS PASSED\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "knowledge", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0114_knowledge_epistemic", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "knowledge", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0119_knowledge_epistemic_hessian_transcendentals", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "knowledge", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0124_knowledge_generic_knowledge", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "knowledge", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0129_knowledge_knowledge_value_with_epistemic", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "effects_algebra", |
| "check_pass": false, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "source: /tmp/sounio-eval-_4fw7nrx/0133_effects_algebra_algebra_observe_synthesis.sio 444\n bytes\ntotal source: 10540\n bytes\ntokens: 2483\n\nst_count: 0\n depth_end: 0\n\nglobals: 0\n bss=1048864\n\neffects: scanned 23\n functions\nrefined: 51\n in 2\n passes\n prev=70\n L1\n\n prev=28\n L2\n\n prev=28\n L3\n\n prev=3\n L6\n\n prev=3\n L6\n\n prev=16\n L16\n\nunc_detail: dot=0\n decl=0\n let=0\n colon=0\n other=6\n\nepistemic: 1442\n expr, 1436\n certain (99\n%), 6\n uncertain [ident=28\n ops=29\n other=6\n], types=7\n, gate=950\n/1000\nepistemic_main: 46\n expr, 40\n certain (86\n%), 6\n uncertain\nknightian: 0\n sites, gap_sum=0\n\nknowledge_subtype: 0\n sites, 0\n violations\nknowledge_units: 0\n sites with dimensional Knowledge<T>\ntier_dist: PLATINUM=1436\n GOLD=0\n SILVER=0\n BRONZE=0\n [PLATINUM%=99\n%]\neconf: total=1442\n platinum=1436\n gate_pass=1436\n knightian=6\n min=0\n mean=995\n gate=950\n\nE200 `x` at line 16\n\nE200 `y` at line 16\n\ntypecheck: failed\n", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0133_effects_algebra_algebra_observe_synthesis", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "effects_algebra", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0138_effects_algebra_effect_chain_propagation", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "effects_algebra", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0143_effects_algebra_wave_a_algebra_struct", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "gpu_octonion", |
| "check_pass": false, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "source: /tmp/sounio-eval-_4fw7nrx/0148_gpu_octonion_octonion_hessian_fano_annotated.sio 473\n bytes\ntotal source: 10569\n bytes\ntokens: 2502\n\nst_count: 0\n depth_end: 0\n\nglobals: 0\n bss=1048864\n\neffects: scanned 23\n functions\nrefined: 54\n in 3\n passes\n prev=70\n L1\n\n prev=28\n L2\n\n prev=28\n L3\n\n prev=8\n L8\n\n prev=17\n L8\n\n prev=17\n L8\n\n prev=16\n L12\n\n prev=16\n L13\n\n prev=28\n L14\n\nunc_detail: dot=0\n decl=0\n let=0\n colon=0\n other=9\n\nepistemic: 1449\n expr, 1438\n certain (99\n%), 11\n uncertain [ident=33\n ops=31\n other=9\n], types=7\n, gate=950\n/1000\nepistemic_main: 53\n expr, 42\n certain (79\n%), 11\n uncertain\nknightian: 0\n sites, gap_sum=0\n\nknowledge_subtype: 0\n sites, 0\n violations\nknowledge_units: 0\n sites with dimensional Knowledge<T>\ntier_dist: PLATINUM=1438\n GOLD=0\n SILVER=0\n BRONZE=0\n [PLATINUM%=99\n%]\neconf: total=1449\n platinum=1438\n gate_pass=1438\n knightian=11\n min=0\n mean=992\n gate=950\n\nE200 `Beta8Phase2` at line 8\n\nE200 `real` at line 8\n\nerror: unknown field access at line 8\n\nerror: unknown field access at line 8\n\nE200 `beta` at line 8\n\nerror: unknown field access at line 8\n\nerror: unknown field access at line 8\n\nE200 `Beta8Phase2` at line 12\n\nE200 `real` at line 12\n\nE200 `beta` at line 12\n\ntypecheck: failed\n", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0148_gpu_octonion_octonion_hessian_fano_annotated", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "reserve_advanced", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0153_reserve_advanced_sieve", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "reserve_advanced", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0158_reserve_advanced_mod_demo", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "reserve_advanced", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0163_reserve_advanced_dissertation_oral_pd_demo", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "reserve_advanced", |
| "check_pass": false, |
| "contract_clean": false, |
| "contract_violations": [ |
| "rust-python:String" |
| ], |
| "diagnostic": "source: /tmp/sounio-eval-_4fw7nrx/0168_reserve_advanced_epistemic_bmi.sio 568\n bytes\ntotal source: 10664\n bytes\ntokens: 2512\n\nst_count: 1\n depth_end: 0\n\nglobals: 0\n bss=1048864\n\neffects: scanned 23\n functions\nrefined: 55\n in 2\n passes\nunc_detail: dot=0\n decl=0\n let=0\n colon=0\n other=0\n\nepistemic: 1451\n expr, 1451\n certain (100\n%), 0\n uncertain [ident=24\n ops=31\n other=0\n], types=7\n, gate=950\n/1000\nepistemic_main: 55\n expr, 55\n certain (100\n%), 0\n uncertain\nknightian: 0\n sites, gap_sum=0\n\nknowledge_subtype: 0\n sites, 0\n violations\nknowledge_units: 0\n sites with dimensional Knowledge<T>\ntier_dist: PLATINUM=1451\n GOLD=0\n SILVER=0\n BRONZE=0\n [PLATINUM%=100\n%]\neconf: total=1451\n platinum=1451\n gate_pass=1451\n knightian=0\n min=1000\n mean=1000\n gate=950\n\nerror: unknown method at line 10\n\ntypecheck: failed\n", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0168_reserve_advanced_epistemic_bmi", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "reserve_advanced", |
| "check_pass": false, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "source: /tmp/sounio-eval-_4fw7nrx/0173_reserve_advanced_gpu_hypercomplex.sio 268\n bytes\ntotal source: 10364\n bytes\ntokens: 2490\n\nst_count: 1\n depth_end: 0\n\nglobals: 0\n bss=1048864\n\neffects: scanned 23\n functions\nrefined: 52\n in 2\n passes\n prev=19\n L4\n\nunc_detail: dot=0\n decl=0\n let=0\n colon=0\n other=1\n\nepistemic: 1448\n expr, 1446\n certain (99\n%), 2\n uncertain [ident=24\n ops=30\n other=1\n], types=7\n, gate=950\n/1000\nepistemic_main: 52\n expr, 50\n certain (96\n%), 2\n uncertain\nknightian: 0\n sites, gap_sum=0\n\nknowledge_subtype: 0\n sites, 0\n violations\nknowledge_units: 0\n sites with dimensional Knowledge<T>\ntier_dist: PLATINUM=1446\n GOLD=0\n SILVER=0\n BRONZE=0\n [PLATINUM%=99\n%]\neconf: total=1448\n platinum=1446\n gate_pass=1446\n knightian=2\n min=0\n mean=998\n gate=950\n\nE200 `d` at line 4\n\ntypecheck: failed\n", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0173_reserve_advanced_gpu_hypercomplex", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "reserve_advanced", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0178_reserve_advanced_hof_fn_ref_call", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "reserve_advanced", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0183_reserve_advanced_struct_mutation", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "reserve_advanced", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0188_reserve_advanced_native_higher_order", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "reserve_advanced", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0193_reserve_advanced_playground_scientific_contract", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "reserve_advanced", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0198_reserve_advanced_uncertainty_ppm", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "reserve_advanced", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0203_reserve_advanced_simple_test", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "reserve_advanced", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0208_reserve_advanced_zd_audit_witness", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "reserve_advanced", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0213_reserve_advanced_test_struct_init", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "reserve_advanced", |
| "check_pass": false, |
| "contract_clean": false, |
| "contract_violations": [ |
| "rust-python:.to_string()", |
| "rust-python:String" |
| ], |
| "diagnostic": "source: /tmp/sounio-eval-_4fw7nrx/0218_reserve_advanced_ontology_multiple_disjoint_known_failure.sio 402\n bytes\ntotal source: 10498\n bytes\ntokens: 2485\n\nst_count: 1\n depth_end: 0\n\nglobals: 0\n bss=1048864\n\neffects: scanned 23\n functions\nrefined: 52\n in 2\n passes\nunc_detail: dot=0\n decl=0\n let=0\n colon=0\n other=0\n\nepistemic: 1438\n expr, 1438\n certain (100\n%), 0\n uncertain [ident=23\n ops=29\n other=0\n], types=7\n, gate=950\n/1000\nepistemic_main: 42\n expr, 42\n certain (100\n%), 0\n uncertain\nknightian: 0\n sites, gap_sum=0\n\nknowledge_subtype: 0\n sites, 0\n violations\nknowledge_units: 0\n sites with dimensional Knowledge<T>\ntier_dist: PLATINUM=1438\n GOLD=0\n SILVER=0\n BRONZE=0\n [PLATINUM%=100\n%]\neconf: total=1438\n platinum=1438\n gate_pass=1438\n knightian=0\n min=1000\n mean=1000\n gate=950\n\nerror: unknown method at line 8\n\ntypecheck: failed\n", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0218_reserve_advanced_ontology_multiple_disjoint_known_failure", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "reserve_advanced", |
| "check_pass": false, |
| "contract_clean": false, |
| "contract_violations": [ |
| "rust-python:.to_string()", |
| "rust-python:String" |
| ], |
| "diagnostic": "source: /tmp/sounio-eval-_4fw7nrx/0223_reserve_advanced_ontology_transitive.sio 495\n bytes\ntotal source: 10591\n bytes\ntokens: 2505\n\nst_count: 1\n depth_end: 0\n\nglobals: 0\n bss=1048864\n\neffects: scanned 23\n functions\nrefined: 52\n in 2\n passes\nunc_detail: dot=0\n decl=0\n let=0\n colon=0\n other=0\n\nepistemic: 1446\n expr, 1446\n certain (100\n%), 0\n uncertain [ident=23\n ops=29\n other=0\n], types=7\n, gate=950\n/1000\nepistemic_main: 50\n expr, 50\n certain (100\n%), 0\n uncertain\nknightian: 0\n sites, gap_sum=0\n\nknowledge_subtype: 0\n sites, 0\n violations\nknowledge_units: 0\n sites with dimensional Knowledge<T>\ntier_dist: PLATINUM=1446\n GOLD=0\n SILVER=0\n BRONZE=0\n [PLATINUM%=100\n%]\neconf: total=1446\n platinum=1446\n gate_pass=1446\n knightian=0\n min=1000\n mean=1000\n gate=950\n\nerror: unknown method at line 8\n\nerror: unknown method at line 8\n\nerror: unknown method at line 8\n\ntypecheck: failed\n", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0223_reserve_advanced_ontology_transitive", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "reserve_advanced", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0228_reserve_advanced_refinement_inline", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "reserve_advanced", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0233_reserve_advanced_refinement_probability", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "reserve_advanced", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "expert", |
| "has_prediction": true, |
| "id": "0240_reserve_advanced_sb_002_effect_safe_division", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "basic", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0243_basic_approx_propagation", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "basic", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0248_basic_closure_capture", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "basic", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0253_basic_closure_sort_by", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "basic", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0258_basic_explicit_deref_field", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "basic", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0263_basic_generics_multi_param", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "basic", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0268_basic_gum_iso_budget_ode", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "basic", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0273_basic_i32_implicit_return", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "basic", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0278_basic_issue11_ptr_index_write_u8", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "basic", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0283_basic_linear_correct_consume", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "basic", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0288_basic_mc_struct_large", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "basic", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0293_basic_ode_bridge_exponential", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "basic", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0298_basic_slice_fat_pointers", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "basic", |
| "check_pass": false, |
| "contract_clean": false, |
| "contract_violations": [ |
| "rust-python:Vec<", |
| "rust-python:vec!" |
| ], |
| "diagnostic": "source: /tmp/sounio-eval-_4fw7nrx/0303_basic_test_nested_struct_return.sio 317\n bytes\ntotal source: 10413\n bytes\ntokens: 2484\n\nst_count: 1\n depth_end: 0\n\nglobals: 0\n bss=1048864\n\neffects: scanned 23\n functions\nrefined: 54\n in 2\n passes\nunc_detail: dot=0\n decl=0\n let=0\n colon=0\n other=0\n\nepistemic: 1436\n expr, 1436\n certain (100\n%), 0\n uncertain [ident=23\n ops=31\n other=0\n], types=7\n, gate=950\n/1000\nepistemic_main: 40\n expr, 40\n certain (100\n%), 0\n uncertain\nknightian: 0\n sites, gap_sum=0\n\nknowledge_subtype: 0\n sites, 0\n violations\nknowledge_units: 0\n sites with dimensional Knowledge<T>\ntier_dist: PLATINUM=1436\n GOLD=0\n SILVER=0\n BRONZE=0\n [PLATINUM%=100\n%]\neconf: total=1436\n platinum=1436\n gate_pass=1436\n knightian=0\n min=1000\n mean=1000\n gate=950\n\nerror: unknown method at line 4\n\nE200 `vec` at line 9\n\ntypecheck: failed\n", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0303_basic_test_nested_struct_return", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "ffi_interop", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0307_ffi_interop_synthetic_0002", |
| "run_output_tail": "4\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "ffi_interop", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0314_ffi_interop_synthetic_0009", |
| "run_output_tail": "16\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "gpu_octonion", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "expert", |
| "has_prediction": true, |
| "id": "0320_gpu_octonion_synthetic_0015", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "ffi_interop", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0328_ffi_interop_synthetic_0023", |
| "run_output_tail": "call\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "effects_algebra", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "expert", |
| "has_prediction": true, |
| "id": "0330_effects_algebra_synthetic_0025", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "knowledge", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0339_knowledge_synthetic_0034", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "gpu_octonion", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0343_gpu_octonion_synthetic_0038", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "effects_algebra", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0346_effects_algebra_synthetic_0041", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "ffi_interop", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0354_ffi_interop_synthetic_0049", |
| "run_output_tail": "16\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "knowledge", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0355_knowledge_synthetic_0050", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "effects_algebra", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0362_effects_algebra_synthetic_0057", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "gpu_octonion", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0367_gpu_octonion_synthetic_0062", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "knowledge", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0371_knowledge_synthetic_0066", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "ffi_interop", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0378_ffi_interop_synthetic_0073", |
| "run_output_tail": "16\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "effects_algebra", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "expert", |
| "has_prediction": true, |
| "id": "0380_effects_algebra_synthetic_0075", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "knowledge", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0389_knowledge_synthetic_0084", |
| "run_output_tail": "usable\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "gpu_octonion", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0393_gpu_octonion_synthetic_0088", |
| "run_output_tail": "8\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "effects_algebra", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0396_effects_algebra_synthetic_0091", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "ffi_interop", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0404_ffi_interop_synthetic_0099", |
| "run_output_tail": "call\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "knowledge", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0405_knowledge_synthetic_0100", |
| "run_output_tail": "usable\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "effects_algebra", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0412_effects_algebra_synthetic_0107", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "gpu_octonion", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0417_gpu_octonion_synthetic_0112", |
| "run_output_tail": "8\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "knowledge", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0421_knowledge_synthetic_0116", |
| "run_output_tail": "usable\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "ffi_interop", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0428_ffi_interop_synthetic_0123", |
| "run_output_tail": "call\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "effects_algebra", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "expert", |
| "has_prediction": true, |
| "id": "0430_effects_algebra_synthetic_0125", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "knowledge", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0439_knowledge_synthetic_0134", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "gpu_octonion", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0443_gpu_octonion_synthetic_0138", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "effects_algebra", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0446_effects_algebra_synthetic_0141", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "ffi_interop", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0454_ffi_interop_synthetic_0149", |
| "run_output_tail": "16\n", |
| "run_pass": false, |
| "runs_expected": true |
| }, |
| { |
| "category": "knowledge", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0455_knowledge_synthetic_0150", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "effects_algebra", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0462_effects_algebra_synthetic_0157", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "gpu_octonion", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0467_gpu_octonion_synthetic_0162", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "knowledge", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0471_knowledge_synthetic_0166", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "ffi_interop", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0478_ffi_interop_synthetic_0173", |
| "run_output_tail": "16\n", |
| "run_pass": false, |
| "runs_expected": true |
| }, |
| { |
| "category": "effects_algebra", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "expert", |
| "has_prediction": true, |
| "id": "0480_effects_algebra_synthetic_0175", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "knowledge", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0489_knowledge_synthetic_0184", |
| "run_output_tail": "usable\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "gpu_octonion", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0493_gpu_octonion_synthetic_0188", |
| "run_output_tail": "4\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "effects_algebra", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0496_effects_algebra_synthetic_0191", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "basic", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0499_basic_synthetic_0194", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "ffi_interop", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0506_ffi_interop_synthetic_0201", |
| "run_output_tail": "16\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "knowledge", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0509_knowledge_synthetic_0204", |
| "run_output_tail": "usable\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "basic", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0516_basic_synthetic_0211", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "effects_algebra", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0518_effects_algebra_synthetic_0213", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "gpu_octonion", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0525_gpu_octonion_synthetic_0220", |
| "run_output_tail": "4\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "knowledge", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0531_knowledge_synthetic_0226", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "basic", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0535_basic_synthetic_0230", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "ffi_interop", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "expert", |
| "has_prediction": true, |
| "id": "0540_ffi_interop_synthetic_0235", |
| "run_output_tail": "call\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "effects_algebra", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0543_effects_algebra_synthetic_0238", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "basic", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "expert", |
| "has_prediction": true, |
| "id": "0550_basic_synthetic_0245", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "knowledge", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0556_knowledge_synthetic_0251", |
| "run_output_tail": "dose-ok\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "gpu_octonion", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0561_gpu_octonion_synthetic_0256", |
| "run_output_tail": "8\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "effects_algebra", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0566_effects_algebra_synthetic_0261", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "basic", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0569_basic_synthetic_0264", |
| "run_output_tail": "odd\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "ffi_interop", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0576_ffi_interop_synthetic_0271", |
| "run_output_tail": "call\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "knowledge", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0579_knowledge_synthetic_0274", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "basic", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0586_basic_synthetic_0281", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "effects_algebra", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0594_effects_algebra_synthetic_0289", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "gpu_octonion", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0595_gpu_octonion_synthetic_0290", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "knowledge", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0598_knowledge_synthetic_0293", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "basic", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0605_basic_synthetic_0300", |
| "run_output_tail": "odd\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "ffi_interop", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "expert", |
| "has_prediction": true, |
| "id": "0610_ffi_interop_synthetic_0305", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "effects_algebra", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0616_effects_algebra_synthetic_0311", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "basic", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "expert", |
| "has_prediction": true, |
| "id": "0620_basic_synthetic_0315", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "knowledge", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0621_knowledge_synthetic_0316", |
| "run_output_tail": "usable\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "gpu_octonion", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0631_gpu_octonion_synthetic_0326", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "basic", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0639_basic_synthetic_0334", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "effects_algebra", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0641_effects_algebra_synthetic_0336", |
| "run_output_tail": "ready\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "knowledge", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0643_knowledge_synthetic_0338", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "ffi_interop", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0646_ffi_interop_synthetic_0341", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "basic", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0656_basic_synthetic_0351", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "effects_algebra", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0664_effects_algebra_synthetic_0359", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "gpu_octonion", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0665_gpu_octonion_synthetic_0360", |
| "run_output_tail": "4\n", |
| "run_pass": false, |
| "runs_expected": true |
| }, |
| { |
| "category": "knowledge", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0668_knowledge_synthetic_0363", |
| "run_output_tail": "dose-ok\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "basic", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0675_basic_synthetic_0370", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "ffi_interop", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "expert", |
| "has_prediction": true, |
| "id": "0680_ffi_interop_synthetic_0375", |
| "run_output_tail": "call\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "effects_algebra", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0686_effects_algebra_synthetic_0381", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "basic", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "expert", |
| "has_prediction": true, |
| "id": "0690_basic_synthetic_0385", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "knowledge", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0691_knowledge_synthetic_0386", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "gpu_octonion", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0701_gpu_octonion_synthetic_0396", |
| "run_output_tail": "4\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "basic", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0709_basic_synthetic_0404", |
| "run_output_tail": "odd\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "effects_algebra", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0711_effects_algebra_synthetic_0406", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "knowledge", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0713_knowledge_synthetic_0408", |
| "run_output_tail": "usable\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "ffi_interop", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0716_ffi_interop_synthetic_0411", |
| "run_output_tail": "call\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "effects_algebra", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0734_effects_algebra_synthetic_0429", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "gpu_octonion", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0735_gpu_octonion_synthetic_0430", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "knowledge", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0738_knowledge_synthetic_0433", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "ffi_interop", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "expert", |
| "has_prediction": true, |
| "id": "0750_ffi_interop_synthetic_0445", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "effects_algebra", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0756_effects_algebra_synthetic_0451", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "knowledge", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0761_knowledge_synthetic_0456", |
| "run_output_tail": "usable\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "gpu_octonion", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0771_gpu_octonion_synthetic_0466", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "effects_algebra", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0781_effects_algebra_synthetic_0476", |
| "run_output_tail": "ready\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "knowledge", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0783_knowledge_synthetic_0478", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "ffi_interop", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0786_ffi_interop_synthetic_0481", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "gpu_octonion", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0791_gpu_octonion_synthetic_0486", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "effects_algebra", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0804_effects_algebra_synthetic_0499", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "knowledge", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0808_knowledge_synthetic_0503", |
| "run_output_tail": "dose-ok\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "ffi_interop", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "expert", |
| "has_prediction": true, |
| "id": "0820_ffi_interop_synthetic_0515", |
| "run_output_tail": "call\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "effects_algebra", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0826_effects_algebra_synthetic_0521", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "gpu_octonion", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0827_gpu_octonion_synthetic_0522", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "knowledge", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0831_knowledge_synthetic_0526", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "reserve_advanced", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0851_reserve_advanced_synthetic_0546", |
| "run_output_tail": "196\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "effects_algebra", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0854_effects_algebra_synthetic_0549", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "knowledge", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0856_knowledge_synthetic_0551", |
| "run_output_tail": "dose-ok\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "ffi_interop", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0859_ffi_interop_synthetic_0554", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "gpu_octonion", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0865_gpu_octonion_synthetic_0560", |
| "run_output_tail": "4\n", |
| "run_pass": false, |
| "runs_expected": true |
| }, |
| { |
| "category": "effects_algebra", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "expert", |
| "has_prediction": true, |
| "id": "0880_effects_algebra_synthetic_0575", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "knowledge", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0885_knowledge_synthetic_0580", |
| "run_output_tail": "usable\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "reserve_advanced", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0892_reserve_advanced_synthetic_0587", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "ffi_interop", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0898_ffi_interop_synthetic_0593", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "gpu_octonion", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0906_gpu_octonion_synthetic_0601", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "reserve_advanced", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0931_reserve_advanced_synthetic_0626", |
| "run_output_tail": "100\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "ffi_interop", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0939_ffi_interop_synthetic_0634", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "gpu_octonion", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0945_gpu_octonion_synthetic_0640", |
| "run_output_tail": "8\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "reserve_advanced", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "0972_reserve_advanced_synthetic_0667", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "ffi_interop", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "0978_ffi_interop_synthetic_0673", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "gpu_octonion", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "0986_gpu_octonion_synthetic_0681", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "reserve_advanced", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "1011_reserve_advanced_synthetic_0706", |
| "run_output_tail": "36\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "ffi_interop", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "1019_ffi_interop_synthetic_0714", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "gpu_octonion", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "1025_gpu_octonion_synthetic_0720", |
| "run_output_tail": "8\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "reserve_advanced", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "1052_reserve_advanced_synthetic_0747", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "ffi_interop", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "1058_ffi_interop_synthetic_0753", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "gpu_octonion", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "1066_gpu_octonion_synthetic_0761", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "reserve_advanced", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "1091_reserve_advanced_synthetic_0786", |
| "run_output_tail": "64\n", |
| "run_pass": false, |
| "runs_expected": true |
| }, |
| { |
| "category": "ffi_interop", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "1099_ffi_interop_synthetic_0794", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "gpu_octonion", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "1105_gpu_octonion_synthetic_0800", |
| "run_output_tail": "8\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "reserve_advanced", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "1132_reserve_advanced_synthetic_0827", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "ffi_interop", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "1138_ffi_interop_synthetic_0833", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "gpu_octonion", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "1146_gpu_octonion_synthetic_0841", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "reserve_advanced", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "1171_reserve_advanced_synthetic_0866", |
| "run_output_tail": "144\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "ffi_interop", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "1179_ffi_interop_synthetic_0874", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "gpu_octonion", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "1185_gpu_octonion_synthetic_0880", |
| "run_output_tail": "8\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "reserve_advanced", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "1212_reserve_advanced_synthetic_0907", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "ffi_interop", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "1218_ffi_interop_synthetic_0913", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "gpu_octonion", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "1226_gpu_octonion_synthetic_0921", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "reserve_advanced", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "1251_reserve_advanced_synthetic_0946", |
| "run_output_tail": "64\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "ffi_interop", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "1259_ffi_interop_synthetic_0954", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "gpu_octonion", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "intermediate", |
| "has_prediction": true, |
| "id": "1265_gpu_octonion_synthetic_0960", |
| "run_output_tail": "8\n", |
| "run_pass": true, |
| "runs_expected": true |
| }, |
| { |
| "category": "reserve_advanced", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "beginner", |
| "has_prediction": true, |
| "id": "1292_reserve_advanced_synthetic_0987", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| }, |
| { |
| "category": "ffi_interop", |
| "check_pass": true, |
| "contract_clean": true, |
| "contract_violations": [], |
| "diagnostic": "", |
| "difficulty": "advanced", |
| "has_prediction": true, |
| "id": "1298_ffi_interop_synthetic_0993", |
| "run_output_tail": null, |
| "run_pass": null, |
| "runs_expected": false |
| } |
| ], |
| "run_expected": 65, |
| "run_expected_predicted": 65, |
| "run_pass": 58, |
| "run_pass_overall_rate": 0.29, |
| "run_rate": 0.8923076923076924, |
| "run_rate_on_predicted": 0.8923076923076924, |
| "total_eval_rows": 200 |
| } |
|
|
