{ "by_category": { "basic": { "check_pass": 30, "check_pass_contract_clean": 30, "contract_clean": 30, "missing_predictions": 0, "rows": 31 }, "effects_algebra": { "check_pass": 29, "check_pass_contract_clean": 29, "contract_clean": 30, "missing_predictions": 0, "rows": 30 }, "ffi_interop": { "check_pass": 31, "check_pass_contract_clean": 31, "contract_clean": 31, "missing_predictions": 0, "rows": 31 }, "gpu_octonion": { "check_pass": 29, "check_pass_contract_clean": 29, "contract_clean": 30, "missing_predictions": 0, "rows": 30 }, "humaneval_mbpp": { "check_pass": 16, "check_pass_contract_clean": 16, "contract_clean": 17, "missing_predictions": 0, "rows": 17 }, "knowledge": { "check_pass": 31, "check_pass_contract_clean": 31, "contract_clean": 31, "missing_predictions": 0, "rows": 31 }, "reserve_advanced": { "check_pass": 26, "check_pass_contract_clean": 26, "contract_clean": 27, "missing_predictions": 0, "rows": 30 } }, "check_pass": 192, "check_pass_contract_clean": 192, "check_pass_contract_clean_rate_on_predictions": 0.96, "check_rate": 0.96, "check_rate_on_predictions": 0.96, "contract_clean": 196, "contract_clean_rate_on_predictions": 0.98, "contract_violation_counts": { "rust-python:.to_string()": 2, "rust-python:String": 3, "rust-python:Vec<": 1, "rust-python:vec!": 1 }, "missing_predictions": 0, "mode": "predictions", "predictions_found": 200, "results": [ { "category": "basic", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0002_basic_algebra_properties_basic", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "basic", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0007_basic_effect_handler_basic", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "basic", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0012_basic_import_basic_a", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "basic", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0017_basic_native_match_int", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "basic", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0022_basic_unobserved_basic", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "humaneval_mbpp", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "expert", "has_prediction": true, "id": "0030_humaneval_mbpp_004_palindrome", "run_output_tail": "004_palindrome: ALL TESTS PASSED\n", "run_pass": true, "runs_expected": true }, { "category": "humaneval_mbpp", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0035_humaneval_mbpp_009_power", "run_output_tail": "009_power: ALL TESTS PASSED\n", "run_pass": true, "runs_expected": true }, { "category": "humaneval_mbpp", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "expert", "has_prediction": true, "id": "0040_humaneval_mbpp_014_rotate_array", "run_output_tail": "014_rotate_array: ALL TESTS PASSED\n", "run_pass": true, "runs_expected": true }, { "category": "humaneval_mbpp", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0045_humaneval_mbpp_019_string_to_int", "run_output_tail": "019_string_to_int: ALL TESTS PASSED\n", "run_pass": true, "runs_expected": true }, { "category": "humaneval_mbpp", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "expert", "has_prediction": true, "id": "0050_humaneval_mbpp_024_histogram", "run_output_tail": "024_histogram: ALL TESTS PASSED\n", "run_pass": true, "runs_expected": true }, { "category": "humaneval_mbpp", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0055_humaneval_mbpp_029_selection_sort", "run_output_tail": "029_selection_sort: ALL TESTS PASSED\n", "run_pass": true, "runs_expected": true }, { "category": "humaneval_mbpp", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "expert", "has_prediction": true, "id": "0060_humaneval_mbpp_034_majority_element", "run_output_tail": "034_majority_element: ALL TESTS PASSED\n", "run_pass": true, "runs_expected": true }, { "category": "humaneval_mbpp", "check_pass": false, "contract_clean": true, "contract_violations": [], "diagnostic": "source: /tmp/sounio-eval-_4fw7nrx/0065_humaneval_mbpp_039_house_robber.sio 412\n bytes\ntotal source: 10508\n bytes\ntokens: 2510\n\nst_count: 0\n depth_end: 0\n\nglobals: 0\n bss=1048864\n\neffects: scanned 23\n functions\nrefined: 54\n in 2\n passes\nunc_detail: dot=0\n decl=0\n let=0\n colon=0\n other=0\n\nepistemic: 1451\n expr, 1451\n certain (100\n%), 0\n uncertain [ident=25\n ops=29\n other=0\n], types=7\n, gate=950\n/1000\nepistemic_main: 55\n expr, 55\n certain (100\n%), 0\n uncertain\nknightian: 0\n sites, gap_sum=0\n\nknowledge_subtype: 0\n sites, 0\n violations\nknowledge_units: 0\n sites with dimensional Knowledge\ntier_dist: PLATINUM=1451\n GOLD=0\n SILVER=0\n BRONZE=0\n [PLATINUM%=100\n%]\neconf: total=1451\n platinum=1451\n gate_pass=1451\n knightian=0\n min=1000\n mean=1000\n gate=950\n\nerror: unknown identifier at line 5\n\ntypecheck: failed\n", "difficulty": "intermediate", "has_prediction": true, "id": "0065_humaneval_mbpp_039_house_robber", "run_output_tail": null, "run_pass": null, "runs_expected": true }, { "category": "humaneval_mbpp", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "expert", "has_prediction": true, "id": "0070_humaneval_mbpp_044_decode_ways", "run_output_tail": "044_decode_ways: ALL TESTS PASSED\n", "run_pass": true, "runs_expected": true }, { "category": "humaneval_mbpp", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0075_humaneval_mbpp_049_count_inversions", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "humaneval_mbpp", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "expert", "has_prediction": true, "id": "0080_humaneval_mbpp_054_valid_bst", "run_output_tail": "054_valid_bst: ALL TESTS PASSED\n", "run_pass": true, "runs_expected": true }, { "category": "humaneval_mbpp", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0085_humaneval_mbpp_060_minimum_window", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "humaneval_mbpp", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "expert", "has_prediction": true, "id": "0090_humaneval_mbpp_066_surrounded_regions", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "humaneval_mbpp", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0095_humaneval_mbpp_073_rabin_karp", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "humaneval_mbpp", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "expert", "has_prediction": true, "id": "0100_humaneval_mbpp_080_search_rotated", "run_output_tail": "080_search_rotated: ALL TESTS PASSED\n", "run_pass": true, "runs_expected": true }, { "category": "humaneval_mbpp", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0105_humaneval_mbpp_085_add_binary", "run_output_tail": "0105_humaneval_mbpp_085_add_binary: ALL TESTS PASSED\n", "run_pass": false, "runs_expected": true }, { "category": "humaneval_mbpp", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "expert", "has_prediction": true, "id": "0110_humaneval_mbpp_092_popcount", "run_output_tail": "092_popcount: ALL TESTS PASSED\n", "run_pass": true, "runs_expected": true }, { "category": "knowledge", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0114_knowledge_epistemic", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "knowledge", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0119_knowledge_epistemic_hessian_transcendentals", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "knowledge", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0124_knowledge_generic_knowledge", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "knowledge", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0129_knowledge_knowledge_value_with_epistemic", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "effects_algebra", "check_pass": false, "contract_clean": true, "contract_violations": [], "diagnostic": "source: /tmp/sounio-eval-_4fw7nrx/0133_effects_algebra_algebra_observe_synthesis.sio 444\n bytes\ntotal source: 10540\n bytes\ntokens: 2483\n\nst_count: 0\n depth_end: 0\n\nglobals: 0\n bss=1048864\n\neffects: scanned 23\n functions\nrefined: 51\n in 2\n passes\n prev=70\n L1\n\n prev=28\n L2\n\n prev=28\n L3\n\n prev=3\n L6\n\n prev=3\n L6\n\n prev=16\n L16\n\nunc_detail: dot=0\n decl=0\n let=0\n colon=0\n other=6\n\nepistemic: 1442\n expr, 1436\n certain (99\n%), 6\n uncertain [ident=28\n ops=29\n other=6\n], types=7\n, gate=950\n/1000\nepistemic_main: 46\n expr, 40\n certain (86\n%), 6\n uncertain\nknightian: 0\n sites, gap_sum=0\n\nknowledge_subtype: 0\n sites, 0\n violations\nknowledge_units: 0\n sites with dimensional Knowledge\ntier_dist: PLATINUM=1436\n GOLD=0\n SILVER=0\n BRONZE=0\n [PLATINUM%=99\n%]\neconf: total=1442\n platinum=1436\n gate_pass=1436\n knightian=6\n min=0\n mean=995\n gate=950\n\nE200 `x` at line 16\n\nE200 `y` at line 16\n\ntypecheck: failed\n", "difficulty": "beginner", "has_prediction": true, "id": "0133_effects_algebra_algebra_observe_synthesis", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "effects_algebra", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0138_effects_algebra_effect_chain_propagation", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "effects_algebra", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0143_effects_algebra_wave_a_algebra_struct", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "gpu_octonion", "check_pass": false, "contract_clean": true, "contract_violations": [], "diagnostic": "source: /tmp/sounio-eval-_4fw7nrx/0148_gpu_octonion_octonion_hessian_fano_annotated.sio 473\n bytes\ntotal source: 10569\n bytes\ntokens: 2502\n\nst_count: 0\n depth_end: 0\n\nglobals: 0\n bss=1048864\n\neffects: scanned 23\n functions\nrefined: 54\n in 3\n passes\n prev=70\n L1\n\n prev=28\n L2\n\n prev=28\n L3\n\n prev=8\n L8\n\n prev=17\n L8\n\n prev=17\n L8\n\n prev=16\n L12\n\n prev=16\n L13\n\n prev=28\n L14\n\nunc_detail: dot=0\n decl=0\n let=0\n colon=0\n other=9\n\nepistemic: 1449\n expr, 1438\n certain (99\n%), 11\n uncertain [ident=33\n ops=31\n other=9\n], types=7\n, gate=950\n/1000\nepistemic_main: 53\n expr, 42\n certain (79\n%), 11\n uncertain\nknightian: 0\n sites, gap_sum=0\n\nknowledge_subtype: 0\n sites, 0\n violations\nknowledge_units: 0\n sites with dimensional Knowledge\ntier_dist: PLATINUM=1438\n GOLD=0\n SILVER=0\n BRONZE=0\n [PLATINUM%=99\n%]\neconf: total=1449\n platinum=1438\n gate_pass=1438\n knightian=11\n min=0\n mean=992\n gate=950\n\nE200 `Beta8Phase2` at line 8\n\nE200 `real` at line 8\n\nerror: unknown field access at line 8\n\nerror: unknown field access at line 8\n\nE200 `beta` at line 8\n\nerror: unknown field access at line 8\n\nerror: unknown field access at line 8\n\nE200 `Beta8Phase2` at line 12\n\nE200 `real` at line 12\n\nE200 `beta` at line 12\n\ntypecheck: failed\n", "difficulty": "advanced", "has_prediction": true, "id": "0148_gpu_octonion_octonion_hessian_fano_annotated", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "reserve_advanced", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0153_reserve_advanced_sieve", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "reserve_advanced", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0158_reserve_advanced_mod_demo", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "reserve_advanced", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0163_reserve_advanced_dissertation_oral_pd_demo", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "reserve_advanced", "check_pass": false, "contract_clean": false, "contract_violations": [ "rust-python:String" ], "diagnostic": "source: /tmp/sounio-eval-_4fw7nrx/0168_reserve_advanced_epistemic_bmi.sio 568\n bytes\ntotal source: 10664\n bytes\ntokens: 2512\n\nst_count: 1\n depth_end: 0\n\nglobals: 0\n bss=1048864\n\neffects: scanned 23\n functions\nrefined: 55\n in 2\n passes\nunc_detail: dot=0\n decl=0\n let=0\n colon=0\n other=0\n\nepistemic: 1451\n expr, 1451\n certain (100\n%), 0\n uncertain [ident=24\n ops=31\n other=0\n], types=7\n, gate=950\n/1000\nepistemic_main: 55\n expr, 55\n certain (100\n%), 0\n uncertain\nknightian: 0\n sites, gap_sum=0\n\nknowledge_subtype: 0\n sites, 0\n violations\nknowledge_units: 0\n sites with dimensional Knowledge\ntier_dist: PLATINUM=1451\n GOLD=0\n SILVER=0\n BRONZE=0\n [PLATINUM%=100\n%]\neconf: total=1451\n platinum=1451\n gate_pass=1451\n knightian=0\n min=1000\n mean=1000\n gate=950\n\nerror: unknown method at line 10\n\ntypecheck: failed\n", "difficulty": "advanced", "has_prediction": true, "id": "0168_reserve_advanced_epistemic_bmi", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "reserve_advanced", "check_pass": false, "contract_clean": true, "contract_violations": [], "diagnostic": "source: /tmp/sounio-eval-_4fw7nrx/0173_reserve_advanced_gpu_hypercomplex.sio 268\n bytes\ntotal source: 10364\n bytes\ntokens: 2490\n\nst_count: 1\n depth_end: 0\n\nglobals: 0\n bss=1048864\n\neffects: scanned 23\n functions\nrefined: 52\n in 2\n passes\n prev=19\n L4\n\nunc_detail: dot=0\n decl=0\n let=0\n colon=0\n other=1\n\nepistemic: 1448\n expr, 1446\n certain (99\n%), 2\n uncertain [ident=24\n ops=30\n other=1\n], types=7\n, gate=950\n/1000\nepistemic_main: 52\n expr, 50\n certain (96\n%), 2\n uncertain\nknightian: 0\n sites, gap_sum=0\n\nknowledge_subtype: 0\n sites, 0\n violations\nknowledge_units: 0\n sites with dimensional Knowledge\ntier_dist: PLATINUM=1446\n GOLD=0\n SILVER=0\n BRONZE=0\n [PLATINUM%=99\n%]\neconf: total=1448\n platinum=1446\n gate_pass=1446\n knightian=2\n min=0\n mean=998\n gate=950\n\nE200 `d` at line 4\n\ntypecheck: failed\n", "difficulty": "beginner", "has_prediction": true, "id": "0173_reserve_advanced_gpu_hypercomplex", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "reserve_advanced", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0178_reserve_advanced_hof_fn_ref_call", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "reserve_advanced", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0183_reserve_advanced_struct_mutation", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "reserve_advanced", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0188_reserve_advanced_native_higher_order", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "reserve_advanced", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0193_reserve_advanced_playground_scientific_contract", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "reserve_advanced", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0198_reserve_advanced_uncertainty_ppm", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "reserve_advanced", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0203_reserve_advanced_simple_test", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "reserve_advanced", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0208_reserve_advanced_zd_audit_witness", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "reserve_advanced", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0213_reserve_advanced_test_struct_init", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "reserve_advanced", "check_pass": false, "contract_clean": false, "contract_violations": [ "rust-python:.to_string()", "rust-python:String" ], "diagnostic": "source: /tmp/sounio-eval-_4fw7nrx/0218_reserve_advanced_ontology_multiple_disjoint_known_failure.sio 402\n bytes\ntotal source: 10498\n bytes\ntokens: 2485\n\nst_count: 1\n depth_end: 0\n\nglobals: 0\n bss=1048864\n\neffects: scanned 23\n functions\nrefined: 52\n in 2\n passes\nunc_detail: dot=0\n decl=0\n let=0\n colon=0\n other=0\n\nepistemic: 1438\n expr, 1438\n certain (100\n%), 0\n uncertain [ident=23\n ops=29\n other=0\n], types=7\n, gate=950\n/1000\nepistemic_main: 42\n expr, 42\n certain (100\n%), 0\n uncertain\nknightian: 0\n sites, gap_sum=0\n\nknowledge_subtype: 0\n sites, 0\n violations\nknowledge_units: 0\n sites with dimensional Knowledge\ntier_dist: PLATINUM=1438\n GOLD=0\n SILVER=0\n BRONZE=0\n [PLATINUM%=100\n%]\neconf: total=1438\n platinum=1438\n gate_pass=1438\n knightian=0\n min=1000\n mean=1000\n gate=950\n\nerror: unknown method at line 8\n\ntypecheck: failed\n", "difficulty": "advanced", "has_prediction": true, "id": "0218_reserve_advanced_ontology_multiple_disjoint_known_failure", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "reserve_advanced", "check_pass": false, "contract_clean": false, "contract_violations": [ "rust-python:.to_string()", "rust-python:String" ], "diagnostic": "source: /tmp/sounio-eval-_4fw7nrx/0223_reserve_advanced_ontology_transitive.sio 495\n bytes\ntotal source: 10591\n bytes\ntokens: 2505\n\nst_count: 1\n depth_end: 0\n\nglobals: 0\n bss=1048864\n\neffects: scanned 23\n functions\nrefined: 52\n in 2\n passes\nunc_detail: dot=0\n decl=0\n let=0\n colon=0\n other=0\n\nepistemic: 1446\n expr, 1446\n certain (100\n%), 0\n uncertain [ident=23\n ops=29\n other=0\n], types=7\n, gate=950\n/1000\nepistemic_main: 50\n expr, 50\n certain (100\n%), 0\n uncertain\nknightian: 0\n sites, gap_sum=0\n\nknowledge_subtype: 0\n sites, 0\n violations\nknowledge_units: 0\n sites with dimensional Knowledge\ntier_dist: PLATINUM=1446\n GOLD=0\n SILVER=0\n BRONZE=0\n [PLATINUM%=100\n%]\neconf: total=1446\n platinum=1446\n gate_pass=1446\n knightian=0\n min=1000\n mean=1000\n gate=950\n\nerror: unknown method at line 8\n\nerror: unknown method at line 8\n\nerror: unknown method at line 8\n\ntypecheck: failed\n", "difficulty": "beginner", "has_prediction": true, "id": "0223_reserve_advanced_ontology_transitive", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "reserve_advanced", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0228_reserve_advanced_refinement_inline", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "reserve_advanced", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0233_reserve_advanced_refinement_probability", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "reserve_advanced", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "expert", "has_prediction": true, "id": "0240_reserve_advanced_sb_002_effect_safe_division", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "basic", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0243_basic_approx_propagation", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "basic", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0248_basic_closure_capture", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "basic", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0253_basic_closure_sort_by", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "basic", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0258_basic_explicit_deref_field", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "basic", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0263_basic_generics_multi_param", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "basic", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0268_basic_gum_iso_budget_ode", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "basic", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0273_basic_i32_implicit_return", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "basic", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0278_basic_issue11_ptr_index_write_u8", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "basic", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0283_basic_linear_correct_consume", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "basic", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0288_basic_mc_struct_large", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "basic", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0293_basic_ode_bridge_exponential", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "basic", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0298_basic_slice_fat_pointers", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "basic", "check_pass": false, "contract_clean": false, "contract_violations": [ "rust-python:Vec<", "rust-python:vec!" ], "diagnostic": "source: /tmp/sounio-eval-_4fw7nrx/0303_basic_test_nested_struct_return.sio 317\n bytes\ntotal source: 10413\n bytes\ntokens: 2484\n\nst_count: 1\n depth_end: 0\n\nglobals: 0\n bss=1048864\n\neffects: scanned 23\n functions\nrefined: 54\n in 2\n passes\nunc_detail: dot=0\n decl=0\n let=0\n colon=0\n other=0\n\nepistemic: 1436\n expr, 1436\n certain (100\n%), 0\n uncertain [ident=23\n ops=31\n other=0\n], types=7\n, gate=950\n/1000\nepistemic_main: 40\n expr, 40\n certain (100\n%), 0\n uncertain\nknightian: 0\n sites, gap_sum=0\n\nknowledge_subtype: 0\n sites, 0\n violations\nknowledge_units: 0\n sites with dimensional Knowledge\ntier_dist: PLATINUM=1436\n GOLD=0\n SILVER=0\n BRONZE=0\n [PLATINUM%=100\n%]\neconf: total=1436\n platinum=1436\n gate_pass=1436\n knightian=0\n min=1000\n mean=1000\n gate=950\n\nerror: unknown method at line 4\n\nE200 `vec` at line 9\n\ntypecheck: failed\n", "difficulty": "beginner", "has_prediction": true, "id": "0303_basic_test_nested_struct_return", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "ffi_interop", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0307_ffi_interop_synthetic_0002", "run_output_tail": "4\n", "run_pass": true, "runs_expected": true }, { "category": "ffi_interop", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0314_ffi_interop_synthetic_0009", "run_output_tail": "16\n", "run_pass": true, "runs_expected": true }, { "category": "gpu_octonion", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "expert", "has_prediction": true, "id": "0320_gpu_octonion_synthetic_0015", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "ffi_interop", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0328_ffi_interop_synthetic_0023", "run_output_tail": "call\n", "run_pass": true, "runs_expected": true }, { "category": "effects_algebra", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "expert", "has_prediction": true, "id": "0330_effects_algebra_synthetic_0025", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "knowledge", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0339_knowledge_synthetic_0034", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "gpu_octonion", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0343_gpu_octonion_synthetic_0038", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "effects_algebra", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0346_effects_algebra_synthetic_0041", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "ffi_interop", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0354_ffi_interop_synthetic_0049", "run_output_tail": "16\n", "run_pass": true, "runs_expected": true }, { "category": "knowledge", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0355_knowledge_synthetic_0050", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "effects_algebra", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0362_effects_algebra_synthetic_0057", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "gpu_octonion", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0367_gpu_octonion_synthetic_0062", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "knowledge", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0371_knowledge_synthetic_0066", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "ffi_interop", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0378_ffi_interop_synthetic_0073", "run_output_tail": "16\n", "run_pass": true, "runs_expected": true }, { "category": "effects_algebra", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "expert", "has_prediction": true, "id": "0380_effects_algebra_synthetic_0075", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "knowledge", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0389_knowledge_synthetic_0084", "run_output_tail": "usable\n", "run_pass": true, "runs_expected": true }, { "category": "gpu_octonion", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0393_gpu_octonion_synthetic_0088", "run_output_tail": "8\n", "run_pass": true, "runs_expected": true }, { "category": "effects_algebra", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0396_effects_algebra_synthetic_0091", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "ffi_interop", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0404_ffi_interop_synthetic_0099", "run_output_tail": "call\n", "run_pass": true, "runs_expected": true }, { "category": "knowledge", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0405_knowledge_synthetic_0100", "run_output_tail": "usable\n", "run_pass": true, "runs_expected": true }, { "category": "effects_algebra", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0412_effects_algebra_synthetic_0107", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "gpu_octonion", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0417_gpu_octonion_synthetic_0112", "run_output_tail": "8\n", "run_pass": true, "runs_expected": true }, { "category": "knowledge", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0421_knowledge_synthetic_0116", "run_output_tail": "usable\n", "run_pass": true, "runs_expected": true }, { "category": "ffi_interop", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0428_ffi_interop_synthetic_0123", "run_output_tail": "call\n", "run_pass": true, "runs_expected": true }, { "category": "effects_algebra", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "expert", "has_prediction": true, "id": "0430_effects_algebra_synthetic_0125", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "knowledge", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0439_knowledge_synthetic_0134", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "gpu_octonion", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0443_gpu_octonion_synthetic_0138", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "effects_algebra", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0446_effects_algebra_synthetic_0141", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "ffi_interop", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0454_ffi_interop_synthetic_0149", "run_output_tail": "16\n", "run_pass": false, "runs_expected": true }, { "category": "knowledge", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0455_knowledge_synthetic_0150", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "effects_algebra", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0462_effects_algebra_synthetic_0157", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "gpu_octonion", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0467_gpu_octonion_synthetic_0162", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "knowledge", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0471_knowledge_synthetic_0166", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "ffi_interop", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0478_ffi_interop_synthetic_0173", "run_output_tail": "16\n", "run_pass": false, "runs_expected": true }, { "category": "effects_algebra", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "expert", "has_prediction": true, "id": "0480_effects_algebra_synthetic_0175", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "knowledge", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0489_knowledge_synthetic_0184", "run_output_tail": "usable\n", "run_pass": true, "runs_expected": true }, { "category": "gpu_octonion", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0493_gpu_octonion_synthetic_0188", "run_output_tail": "4\n", "run_pass": true, "runs_expected": true }, { "category": "effects_algebra", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0496_effects_algebra_synthetic_0191", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "basic", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0499_basic_synthetic_0194", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "ffi_interop", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0506_ffi_interop_synthetic_0201", "run_output_tail": "16\n", "run_pass": true, "runs_expected": true }, { "category": "knowledge", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0509_knowledge_synthetic_0204", "run_output_tail": "usable\n", "run_pass": true, "runs_expected": true }, { "category": "basic", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0516_basic_synthetic_0211", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "effects_algebra", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0518_effects_algebra_synthetic_0213", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "gpu_octonion", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0525_gpu_octonion_synthetic_0220", "run_output_tail": "4\n", "run_pass": true, "runs_expected": true }, { "category": "knowledge", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0531_knowledge_synthetic_0226", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "basic", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0535_basic_synthetic_0230", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "ffi_interop", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "expert", "has_prediction": true, "id": "0540_ffi_interop_synthetic_0235", "run_output_tail": "call\n", "run_pass": true, "runs_expected": true }, { "category": "effects_algebra", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0543_effects_algebra_synthetic_0238", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "basic", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "expert", "has_prediction": true, "id": "0550_basic_synthetic_0245", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "knowledge", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0556_knowledge_synthetic_0251", "run_output_tail": "dose-ok\n", "run_pass": true, "runs_expected": true }, { "category": "gpu_octonion", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0561_gpu_octonion_synthetic_0256", "run_output_tail": "8\n", "run_pass": true, "runs_expected": true }, { "category": "effects_algebra", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0566_effects_algebra_synthetic_0261", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "basic", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0569_basic_synthetic_0264", "run_output_tail": "odd\n", "run_pass": true, "runs_expected": true }, { "category": "ffi_interop", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0576_ffi_interop_synthetic_0271", "run_output_tail": "call\n", "run_pass": true, "runs_expected": true }, { "category": "knowledge", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0579_knowledge_synthetic_0274", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "basic", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0586_basic_synthetic_0281", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "effects_algebra", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0594_effects_algebra_synthetic_0289", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "gpu_octonion", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0595_gpu_octonion_synthetic_0290", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "knowledge", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0598_knowledge_synthetic_0293", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "basic", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0605_basic_synthetic_0300", "run_output_tail": "odd\n", "run_pass": true, "runs_expected": true }, { "category": "ffi_interop", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "expert", "has_prediction": true, "id": "0610_ffi_interop_synthetic_0305", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "effects_algebra", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0616_effects_algebra_synthetic_0311", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "basic", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "expert", "has_prediction": true, "id": "0620_basic_synthetic_0315", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "knowledge", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0621_knowledge_synthetic_0316", "run_output_tail": "usable\n", "run_pass": true, "runs_expected": true }, { "category": "gpu_octonion", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0631_gpu_octonion_synthetic_0326", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "basic", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0639_basic_synthetic_0334", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "effects_algebra", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0641_effects_algebra_synthetic_0336", "run_output_tail": "ready\n", "run_pass": true, "runs_expected": true }, { "category": "knowledge", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0643_knowledge_synthetic_0338", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "ffi_interop", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0646_ffi_interop_synthetic_0341", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "basic", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0656_basic_synthetic_0351", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "effects_algebra", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0664_effects_algebra_synthetic_0359", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "gpu_octonion", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0665_gpu_octonion_synthetic_0360", "run_output_tail": "4\n", "run_pass": false, "runs_expected": true }, { "category": "knowledge", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0668_knowledge_synthetic_0363", "run_output_tail": "dose-ok\n", "run_pass": true, "runs_expected": true }, { "category": "basic", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0675_basic_synthetic_0370", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "ffi_interop", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "expert", "has_prediction": true, "id": "0680_ffi_interop_synthetic_0375", "run_output_tail": "call\n", "run_pass": true, "runs_expected": true }, { "category": "effects_algebra", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0686_effects_algebra_synthetic_0381", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "basic", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "expert", "has_prediction": true, "id": "0690_basic_synthetic_0385", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "knowledge", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0691_knowledge_synthetic_0386", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "gpu_octonion", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0701_gpu_octonion_synthetic_0396", "run_output_tail": "4\n", "run_pass": true, "runs_expected": true }, { "category": "basic", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0709_basic_synthetic_0404", "run_output_tail": "odd\n", "run_pass": true, "runs_expected": true }, { "category": "effects_algebra", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0711_effects_algebra_synthetic_0406", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "knowledge", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0713_knowledge_synthetic_0408", "run_output_tail": "usable\n", "run_pass": true, "runs_expected": true }, { "category": "ffi_interop", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0716_ffi_interop_synthetic_0411", "run_output_tail": "call\n", "run_pass": true, "runs_expected": true }, { "category": "effects_algebra", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0734_effects_algebra_synthetic_0429", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "gpu_octonion", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0735_gpu_octonion_synthetic_0430", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "knowledge", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0738_knowledge_synthetic_0433", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "ffi_interop", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "expert", "has_prediction": true, "id": "0750_ffi_interop_synthetic_0445", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "effects_algebra", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0756_effects_algebra_synthetic_0451", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "knowledge", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0761_knowledge_synthetic_0456", "run_output_tail": "usable\n", "run_pass": true, "runs_expected": true }, { "category": "gpu_octonion", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0771_gpu_octonion_synthetic_0466", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "effects_algebra", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0781_effects_algebra_synthetic_0476", "run_output_tail": "ready\n", "run_pass": true, "runs_expected": true }, { "category": "knowledge", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0783_knowledge_synthetic_0478", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "ffi_interop", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0786_ffi_interop_synthetic_0481", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "gpu_octonion", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0791_gpu_octonion_synthetic_0486", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "effects_algebra", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0804_effects_algebra_synthetic_0499", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "knowledge", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0808_knowledge_synthetic_0503", "run_output_tail": "dose-ok\n", "run_pass": true, "runs_expected": true }, { "category": "ffi_interop", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "expert", "has_prediction": true, "id": "0820_ffi_interop_synthetic_0515", "run_output_tail": "call\n", "run_pass": true, "runs_expected": true }, { "category": "effects_algebra", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0826_effects_algebra_synthetic_0521", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "gpu_octonion", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0827_gpu_octonion_synthetic_0522", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "knowledge", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0831_knowledge_synthetic_0526", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "reserve_advanced", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0851_reserve_advanced_synthetic_0546", "run_output_tail": "196\n", "run_pass": true, "runs_expected": true }, { "category": "effects_algebra", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0854_effects_algebra_synthetic_0549", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "knowledge", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0856_knowledge_synthetic_0551", "run_output_tail": "dose-ok\n", "run_pass": true, "runs_expected": true }, { "category": "ffi_interop", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0859_ffi_interop_synthetic_0554", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "gpu_octonion", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0865_gpu_octonion_synthetic_0560", "run_output_tail": "4\n", "run_pass": false, "runs_expected": true }, { "category": "effects_algebra", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "expert", "has_prediction": true, "id": "0880_effects_algebra_synthetic_0575", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "knowledge", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0885_knowledge_synthetic_0580", "run_output_tail": "usable\n", "run_pass": true, "runs_expected": true }, { "category": "reserve_advanced", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0892_reserve_advanced_synthetic_0587", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "ffi_interop", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0898_ffi_interop_synthetic_0593", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "gpu_octonion", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0906_gpu_octonion_synthetic_0601", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "reserve_advanced", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0931_reserve_advanced_synthetic_0626", "run_output_tail": "100\n", "run_pass": true, "runs_expected": true }, { "category": "ffi_interop", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0939_ffi_interop_synthetic_0634", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "gpu_octonion", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0945_gpu_octonion_synthetic_0640", "run_output_tail": "8\n", "run_pass": true, "runs_expected": true }, { "category": "reserve_advanced", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "0972_reserve_advanced_synthetic_0667", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "ffi_interop", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "0978_ffi_interop_synthetic_0673", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "gpu_octonion", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "0986_gpu_octonion_synthetic_0681", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "reserve_advanced", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "1011_reserve_advanced_synthetic_0706", "run_output_tail": "36\n", "run_pass": true, "runs_expected": true }, { "category": "ffi_interop", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "1019_ffi_interop_synthetic_0714", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "gpu_octonion", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "1025_gpu_octonion_synthetic_0720", "run_output_tail": "8\n", "run_pass": true, "runs_expected": true }, { "category": "reserve_advanced", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "1052_reserve_advanced_synthetic_0747", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "ffi_interop", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "1058_ffi_interop_synthetic_0753", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "gpu_octonion", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "1066_gpu_octonion_synthetic_0761", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "reserve_advanced", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "1091_reserve_advanced_synthetic_0786", "run_output_tail": "64\n", "run_pass": false, "runs_expected": true }, { "category": "ffi_interop", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "1099_ffi_interop_synthetic_0794", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "gpu_octonion", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "1105_gpu_octonion_synthetic_0800", "run_output_tail": "8\n", "run_pass": true, "runs_expected": true }, { "category": "reserve_advanced", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "1132_reserve_advanced_synthetic_0827", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "ffi_interop", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "1138_ffi_interop_synthetic_0833", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "gpu_octonion", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "1146_gpu_octonion_synthetic_0841", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "reserve_advanced", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "1171_reserve_advanced_synthetic_0866", "run_output_tail": "144\n", "run_pass": true, "runs_expected": true }, { "category": "ffi_interop", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "1179_ffi_interop_synthetic_0874", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "gpu_octonion", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "1185_gpu_octonion_synthetic_0880", "run_output_tail": "8\n", "run_pass": true, "runs_expected": true }, { "category": "reserve_advanced", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "1212_reserve_advanced_synthetic_0907", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "ffi_interop", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "1218_ffi_interop_synthetic_0913", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "gpu_octonion", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "1226_gpu_octonion_synthetic_0921", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "reserve_advanced", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "1251_reserve_advanced_synthetic_0946", "run_output_tail": "64\n", "run_pass": true, "runs_expected": true }, { "category": "ffi_interop", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "1259_ffi_interop_synthetic_0954", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "gpu_octonion", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "intermediate", "has_prediction": true, "id": "1265_gpu_octonion_synthetic_0960", "run_output_tail": "8\n", "run_pass": true, "runs_expected": true }, { "category": "reserve_advanced", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "beginner", "has_prediction": true, "id": "1292_reserve_advanced_synthetic_0987", "run_output_tail": null, "run_pass": null, "runs_expected": false }, { "category": "ffi_interop", "check_pass": true, "contract_clean": true, "contract_violations": [], "diagnostic": "", "difficulty": "advanced", "has_prediction": true, "id": "1298_ffi_interop_synthetic_0993", "run_output_tail": null, "run_pass": null, "runs_expected": false } ], "run_expected": 65, "run_expected_predicted": 65, "run_pass": 58, "run_pass_overall_rate": 0.29, "run_rate": 0.8923076923076924, "run_rate_on_predicted": 0.8923076923076924, "total_eval_rows": 200 }