{ "source": "/workspace/svd_safety/results/basis_sharing/cells/l2_basis_remove40/checkpoint", "upstream": "TUDa-HWAI/Basis_Sharing", "step": "basis_sharing", "ratio_removed": 0.4, "group_size": 2, "nsamples": 256, "seqlen": 2048, "seed": 42, "share_part": [ "v", "k", "q", "up", "gate" ], "private_part": [ "down", "o" ], "factored": "/workspace/svd_safety/results/basis_sharing/cells/l2_basis_remove40/truncated/factored.pt", "param_fraction": 0.5999091746275906, "ranks": { "0.q_proj": 1638, "0.k_proj": 1638, "0.v_proj": 1638, "0.o_proj": 1228, "0.gate_proj": 2072, "0.up_proj": 2072, "0.down_proj": 1791, "1.q_proj": 1638, "1.k_proj": 1638, "1.v_proj": 1638, "1.o_proj": 1228, "1.gate_proj": 2072, "1.up_proj": 2072, "1.down_proj": 1791, "2.q_proj": 1638, "2.k_proj": 1638, "2.v_proj": 1638, "2.o_proj": 1228, "2.gate_proj": 2072, "2.up_proj": 2072, "2.down_proj": 1791, "3.q_proj": 1638, "3.k_proj": 1638, "3.v_proj": 1638, "3.o_proj": 1228, "3.gate_proj": 2072, "3.up_proj": 2072, "3.down_proj": 1791, "4.q_proj": 1638, "4.k_proj": 1638, "4.v_proj": 1638, "4.o_proj": 1228, "4.gate_proj": 2072, "4.up_proj": 2072, "4.down_proj": 1791, "5.q_proj": 1638, "5.k_proj": 1638, "5.v_proj": 1638, "5.o_proj": 1228, "5.gate_proj": 2072, "5.up_proj": 2072, "5.down_proj": 1791, "6.q_proj": 1638, "6.k_proj": 1638, "6.v_proj": 1638, "6.o_proj": 1228, "6.gate_proj": 2072, "6.up_proj": 2072, "6.down_proj": 1791, "7.q_proj": 1638, "7.k_proj": 1638, "7.v_proj": 1638, "7.o_proj": 1228, "7.gate_proj": 2072, "7.up_proj": 2072, "7.down_proj": 1791, "8.q_proj": 1638, "8.k_proj": 1638, "8.v_proj": 1638, "8.o_proj": 1228, "8.gate_proj": 2072, "8.up_proj": 2072, "8.down_proj": 1791, "9.q_proj": 1638, "9.k_proj": 1638, "9.v_proj": 1638, "9.o_proj": 1228, "9.gate_proj": 2072, "9.up_proj": 2072, "9.down_proj": 1791, "10.q_proj": 1638, "10.k_proj": 1638, "10.v_proj": 1638, "10.o_proj": 1228, "10.gate_proj": 2072, "10.up_proj": 2072, "10.down_proj": 1791, "11.q_proj": 1638, "11.k_proj": 1638, "11.v_proj": 1638, "11.o_proj": 1228, "11.gate_proj": 2072, "11.up_proj": 2072, "11.down_proj": 1791, "12.q_proj": 1638, "12.k_proj": 1638, "12.v_proj": 1638, "12.o_proj": 1228, "12.gate_proj": 2072, "12.up_proj": 2072, "12.down_proj": 1791, "13.q_proj": 1638, "13.k_proj": 1638, "13.v_proj": 1638, "13.o_proj": 1228, "13.gate_proj": 2072, "13.up_proj": 2072, "13.down_proj": 1791, "14.q_proj": 1638, "14.k_proj": 1638, "14.v_proj": 1638, "14.o_proj": 1228, "14.gate_proj": 2072, "14.up_proj": 2072, "14.down_proj": 1791, "15.q_proj": 1638, "15.k_proj": 1638, "15.v_proj": 1638, "15.o_proj": 1228, "15.gate_proj": 2072, "15.up_proj": 2072, "15.down_proj": 1791, "16.q_proj": 1638, "16.k_proj": 1638, "16.v_proj": 1638, "16.o_proj": 1228, "16.gate_proj": 2072, "16.up_proj": 2072, "16.down_proj": 1791, "17.q_proj": 1638, "17.k_proj": 1638, "17.v_proj": 1638, "17.o_proj": 1228, "17.gate_proj": 2072, "17.up_proj": 2072, "17.down_proj": 1791, "18.q_proj": 1638, "18.k_proj": 1638, "18.v_proj": 1638, "18.o_proj": 1228, "18.gate_proj": 2072, "18.up_proj": 2072, "18.down_proj": 1791, "19.q_proj": 1638, "19.k_proj": 1638, "19.v_proj": 1638, "19.o_proj": 1228, "19.gate_proj": 2072, "19.up_proj": 2072, "19.down_proj": 1791, "20.q_proj": 1638, "20.k_proj": 1638, "20.v_proj": 1638, "20.o_proj": 1228, "20.gate_proj": 2072, "20.up_proj": 2072, "20.down_proj": 1791, "21.q_proj": 1638, "21.k_proj": 1638, "21.v_proj": 1638, "21.o_proj": 1228, "21.gate_proj": 2072, "21.up_proj": 2072, "21.down_proj": 1791, "22.q_proj": 1638, "22.k_proj": 1638, "22.v_proj": 1638, "22.o_proj": 1228, "22.gate_proj": 2072, "22.up_proj": 2072, "22.down_proj": 1791, "23.q_proj": 1638, "23.k_proj": 1638, "23.v_proj": 1638, "23.o_proj": 1228, "23.gate_proj": 2072, "23.up_proj": 2072, "23.down_proj": 1791, "24.q_proj": 1638, "24.k_proj": 1638, "24.v_proj": 1638, "24.o_proj": 1228, "24.gate_proj": 2072, "24.up_proj": 2072, "24.down_proj": 1791, "25.q_proj": 1638, "25.k_proj": 1638, "25.v_proj": 1638, "25.o_proj": 1228, "25.gate_proj": 2072, "25.up_proj": 2072, "25.down_proj": 1791, "26.q_proj": 1638, "26.k_proj": 1638, "26.v_proj": 1638, "26.o_proj": 1228, "26.gate_proj": 2072, "26.up_proj": 2072, "26.down_proj": 1791, "27.q_proj": 1638, "27.k_proj": 1638, "27.v_proj": 1638, "27.o_proj": 1228, "27.gate_proj": 2072, "27.up_proj": 2072, "27.down_proj": 1791, "28.q_proj": 1638, "28.k_proj": 1638, "28.v_proj": 1638, "28.o_proj": 1228, "28.gate_proj": 2072, "28.up_proj": 2072, "28.down_proj": 1791, "29.q_proj": 1638, "29.k_proj": 1638, "29.v_proj": 1638, "29.o_proj": 1228, "29.gate_proj": 2072, "29.up_proj": 2072, "29.down_proj": 1791, "30.q_proj": 1638, "30.k_proj": 1638, "30.v_proj": 1638, "30.o_proj": 1228, "30.gate_proj": 2072, "30.up_proj": 2072, "30.down_proj": 1791, "31.q_proj": 1638, "31.k_proj": 1638, "31.v_proj": 1638, "31.o_proj": 1228, "31.gate_proj": 2072, "31.up_proj": 2072, "31.down_proj": 1791 }, "stage": "basis_recovery_finetuned", "recovery_finetune": { "protocol": "basis_sharing_coefficient_lora", "script": "gitclones/SVD-LLM/utils/LoRA.py", "data_path": "alpaca-cleaned (/workspace/svd_safety/data/alpaca_cleaned.json)", "epochs": 2, "learning_rate": 0.0001, "batch_size": 64, "micro_batch_size": 4, "cutoff_len": 256, "val_set_size": 2000, "max_train_samples": "all", "precision": "bf16 (upstream hard-codes fp16; see run_with_torch_compat)", "is_methods_own_stage2": false, "note": "Basis Sharing publishes its own LoRA (lora.py: wikitext, batch 1, peft default q/v coefficient targets). This cell uses the project's SVD-LLM alpaca recipe so recovery data is constant across compressors, on all seven coefficient types and never on a basis.", "lora_r": 8, "lora_alpha": 16, "lora_dropout": 0.05, "lora_targets": [ "down_proj", "gate_proj", "k_proj", "o_proj", "q_proj", "up_proj", "v_proj" ], "trained": "Coefficient modules only; shared and private bases frozen", "merge": "C' = C + (alpha/r) B_lora A_lora, then W = C' @ basis", "mean_rel_delta_coefficient": 0.02975503941498963 }, "param_fraction_is_upper_bound": false, "method": "basis_sharing+rescue_iter_swap", "arm": "swapdiscnet_iter", "objective": "disc", "swap_value": "net", "unit": "basis sharing (projection, layer group) component: shared basis row + every member's coefficient column", "insert_factors": "group decomposition store (compression's discarded components)", "evict_factors": "LoRA-merged coefficients x frozen basis (current model)", "store": "/workspace/svd_safety/assets/basis_share_store/llama2_7b_chat_n256_s42_g2", "budget_frac": 0.01, "chunk_frac": 0.001, "rounds_requested": 10, "nu": 1.0, "tau": Infinity, "orr_ceiling": Infinity, "gap_target": null, "dsafe_target": null, "jbb_in_sample": true, "calib_pair_indexes": [ "59", "21", "56", "18", "33", "42", "50", "27", "92", "71", "2", "25", "24", "52", "81", "99", "51", "61", "4", "97", "94", "37", "28", "80", "7", "57", "79", "26", "76", "78", "38", "0", "86", "46", "10", "29", "70", "45", "17", "39", "91", "40", "89", "5", "68", "1", "95", "84", "63", "3", "83", "93", "20", "87", "43", "73", "32", "75", "23", "96", "69", "15", "48", "55", "60", "9", "62", "31", "72", "16", "44", "82", "67", "34", "30", "58", "74", "88", "98", "54", "49", "6", "90", "11", "85", "41", "22", "19", "35", "64", "47", "12", "53", "14", "13", "66", "36", "65", "77", "8" ], "refuse_lead_tokens": 25, "answer_lead_tokens": 32, "insert_scale": 1.0, "evict_top": 512, "max_cnorm": 0.25, "swap": true, "swap_evict": "sigma", "gross_params_inserted": 12948736, "n_components": 924, "n_swapped_out": 924, "components": { "unit_gate_proj_g28": [ 2308, 2389, 2433, 2437, 2527, 3158, 4085 ], "unit_o_proj_g09": [ 1286, 1288, 1294, 1301, 1367, 1376, 1503, 1541, 1642, 1659, 1667, 1717, 1733, 1924, 2196, 2329, 2904, 2991 ], "unit_o_proj_g15": [ 1284, 1286, 1395, 1497, 1516, 1527, 1577, 1636, 1765, 1826, 2137, 2394, 2462 ], "unit_o_proj_g10": [ 1243, 1245, 1264, 1301, 1369, 1381, 1461, 1610, 1768, 1889, 1992, 2173, 2342 ], "unit_v_proj_g30": [ 1753, 2109, 2112, 2301, 2451, 2601, 3135, 3340, 4095 ], "unit_v_proj_g10": [ 1670, 1745, 1784, 1785, 1841, 1844, 1888, 1890, 1957, 1973, 1981, 1994, 2047, 2196, 2231, 2266, 2345, 2402, 2457, 2575, 2609, 2762, 2778, 2802, 2994, 3694 ], "unit_o_proj_g21": [ 1274, 1297, 1372, 1377, 1410, 1911, 1917, 1918, 2042, 2080, 2119, 2415 ], "unit_o_proj_g06": [ 1229, 1282, 1359, 1389, 1426, 1445, 1463, 1479, 1534, 1701, 1719, 1933, 2048, 2498 ], "unit_v_proj_g08": [ 1698, 2147, 2736, 2982 ], "unit_down_proj_g15": [ 1806, 1928, 2079, 2092, 2198, 2203, 2207, 2277, 2302, 2434, 2461, 2594, 2608, 2661, 2735, 2825, 2890, 2945, 3124, 3736 ], "unit_down_proj_g13": [ 1857, 2004, 2211, 2652, 3341 ], "unit_down_proj_g09": [ 1800, 1814, 1817, 1880, 1975, 2013, 2018, 2044, 2108, 2305, 2346, 2380, 2504, 2518, 2802, 2916, 2931, 3316, 3524 ], "unit_down_proj_g05": [ 1809, 1832, 1849, 1861, 1893, 1945, 2009, 2133, 2150, 2195, 2225, 2423, 2593, 2691, 2800, 2956, 3153 ], "unit_v_proj_g14": [ 1796, 1932 ], "unit_o_proj_g28": [ 1270, 1273, 1277, 1307, 1319, 1459, 1504, 1538, 1600, 1605, 1631, 1649, 1736, 1755, 1763, 1873, 1981, 1985, 2146, 2966 ], "unit_o_proj_g08": [ 1228, 1230, 1237, 1279, 1287, 1335, 1373, 1377, 1378, 1381, 1385, 1466, 1505, 1584, 1721, 1736, 1758, 1797, 1804, 1848, 1922, 2048, 2138, 2257 ], "unit_o_proj_g20": [ 1357, 1440, 1516, 1674, 1693 ], "unit_v_proj_g04": [ 1678, 1739, 2056, 2213, 2226, 2237, 2312, 2357, 2359, 2375, 2557, 2581, 2593, 2622, 2820, 2824, 2951, 2960, 2974, 2975, 3005, 3006, 3098, 3122, 3138, 3147, 3261, 3324, 3351, 3406 ], "unit_v_proj_g12": [ 1750, 1751, 1830, 1831, 1843, 1898, 1945, 2059, 2193, 2230, 2314, 2574, 2632, 2670, 2842, 2970, 3487 ], "unit_o_proj_g29": [ 1275, 1322 ], "unit_v_proj_g28": [ 1657, 1815, 1845, 2255, 2627, 2903, 3557, 4091 ], "unit_v_proj_g20": [ 1801, 1923, 2195, 2270, 2513, 2688 ], "unit_v_proj_g18": [ 1650, 1688, 1992, 2022, 2116, 2252, 2431, 2435 ], "unit_down_proj_g07": [ 1858, 1884, 1967, 2048, 2057, 2110, 2591 ], "unit_o_proj_g01": [ 1253 ], "unit_o_proj_g11": [ 1244, 1249, 1258, 1333, 1347, 1460, 1465, 1486, 1520, 1521, 1571, 1580, 2013 ], "unit_v_proj_g06": [ 1687, 1712, 1749, 1845, 1942, 2233, 2423, 2457, 2501, 2567, 2694, 2893, 2946, 3110, 3157, 3264, 3527, 3665 ], "unit_up_proj_g14": [ 2126, 2231, 2243, 2273, 2339, 2534, 2539, 2544, 2829, 2996, 3047, 3258, 3909, 4074 ], "unit_down_proj_g06": [ 2002, 2004 ], "unit_o_proj_g16": [ 1267, 1276, 1302, 1330, 1383, 1413, 1458, 1472, 1504, 1658, 1741, 1802, 2411 ], "unit_down_proj_g12": [ 1858, 1909, 1976, 1999, 2025, 2046, 2091, 2117, 2219, 2228, 2336, 2430, 2436, 2534, 2671, 2674, 2683, 2722, 2752 ], "unit_o_proj_g03": [ 1243, 1250, 1264, 1276, 1408, 1501, 1561, 1609, 1976, 2525 ], "unit_v_proj_g24": [ 1882, 1883, 1913, 1944, 2956 ], "unit_down_proj_g17": [ 1855, 1910, 1920, 2086, 2111, 2182, 2209, 2248, 2379, 2470, 2592, 2605, 2611, 2667, 2718, 2777, 2925, 2939, 2992, 3188, 3349 ], "unit_down_proj_g18": [ 1813, 1849, 1861, 2011, 2182, 2261, 2397, 2422, 2582, 3400, 3574 ], "unit_o_proj_g25": [ 1283, 1308, 1313, 1482, 2107 ], "unit_down_proj_g16": [ 1857, 1867, 1903, 2119, 2182, 2210, 2213, 2377, 2759 ], "unit_o_proj_g13": [ 1453, 1466, 1644, 2079 ], "unit_up_proj_g08": [ 2906 ], "unit_down_proj_g10": [ 1796, 1909, 1981, 1989 ], "unit_down_proj_g11": [ 1809, 1930, 2005, 2031, 2091, 2216, 2463, 2687, 2767 ], "unit_gate_proj_g12": [ 2254, 2268, 2280, 2601 ], "unit_down_proj_g08": [ 1805, 1953, 1960, 2200, 2473 ], "unit_o_proj_g14": [ 1230, 1264, 1270, 1280, 1287, 1331, 1367, 1460, 1478, 1483, 1657, 1663, 2004, 2257 ], "unit_up_proj_g16": [ 3002 ], "unit_down_proj_g14": [ 1826, 1863, 1928 ], "unit_q_proj_g16": [ 1655, 1961, 2061, 2215, 3080 ], "unit_v_proj_g02": [ 1750, 2237, 2255, 2754, 3048, 3487, 3529, 3643, 3817, 3996, 4016, 4019, 4021, 4071 ], "unit_up_proj_g20": [ 2213, 2241, 2376, 2572, 3102, 3787 ], "unit_down_proj_g19": [ 1843, 1858, 1903, 1936, 1950, 2255, 2490, 2864, 2893 ], "unit_down_proj_g21": [ 1918, 2042, 2104, 2166, 3396, 3423 ], "unit_down_proj_g20": [ 1845, 1862, 1893, 1958, 1961, 1971, 2011, 2345, 2458, 2489, 2623, 2782 ], "unit_down_proj_g01": [ 1822, 1863, 1874 ], "unit_v_proj_g26": [ 1698, 1766, 1784, 1795, 1825, 1831, 1972, 2011, 2036, 2103, 2125, 2181, 2194, 2346, 2501, 2513, 2558, 2965, 3009, 3512 ], "unit_o_proj_g18": [ 1277, 1290, 1322, 1329, 1342, 1411, 1455, 1458, 1492, 1582, 1763, 1769, 1897, 1948, 1970, 2010, 2014 ], "unit_gate_proj_g14": [ 2106, 2128, 2538, 2564, 2565, 2584, 2705, 2769, 3072, 3396, 3503 ], "unit_o_proj_g30": [ 1240, 1245, 1300, 1327, 1567, 1678, 1805, 2159 ], "unit_up_proj_g10": [ 2090, 2236, 2287, 2338, 2525, 2646, 2666, 3219, 3355, 3565, 3603, 4047 ], "unit_down_proj_g26": [ 2070, 2190, 2366, 3048 ], "unit_o_proj_g07": [ 1234, 1244, 1285, 1293, 1304, 1331, 1350, 1372, 1393, 1421, 1443, 1460, 1485, 1489, 1561, 1601, 1753, 1758, 1780, 1785, 1786, 1940, 2038, 2194, 2326, 2361, 2423, 2610, 2801 ], "unit_down_proj_g31": [ 2278 ], "unit_down_proj_g04": [ 1794, 1843, 1911, 1957, 1999, 2005, 2026, 2031, 2044, 2092, 2129, 2142, 2173, 2202, 2231, 2233, 2453, 2583, 2597, 2666, 2758 ], "unit_v_proj_g22": [ 2035, 2077, 2316, 2340, 2715, 2716 ], "unit_up_proj_g02": [ 2141, 2197, 2401 ], "unit_gate_proj_g08": [ 2152, 2221, 2300, 2458, 2620, 2693, 2896 ], "unit_o_proj_g24": [ 1384 ], "unit_down_proj_g29": [ 1833, 1858, 1874, 2009, 2087, 2308, 2333, 2339 ], "unit_up_proj_g12": [ 2329, 2617, 3057, 3744 ], "unit_up_proj_g18": [ 2078, 2135, 2141, 2272, 2459, 2480, 2571, 2605, 2705, 2779, 3401 ], "unit_down_proj_g27": [ 1813, 2104 ], "unit_down_proj_g28": [ 1808, 1828, 3146 ], "unit_o_proj_g02": [ 1273, 1276, 1287, 1467, 1548, 1676, 2197 ], "unit_up_proj_g24": [ 2371 ], "unit_o_proj_g17": [ 1233, 1426, 1464, 1486, 1492, 1498, 1524, 1528, 1571, 1608, 1776, 1901, 2226 ], "unit_gate_proj_g06": [ 2456, 2531, 2706, 2837, 3053, 4081 ], "unit_q_proj_g06": [ 1772 ], "unit_o_proj_g26": [ 1523, 1535, 1643, 1774, 2459, 2886 ], "unit_gate_proj_g18": [ 2321, 2429, 2624 ], "unit_down_proj_g25": [ 1908, 1995, 2006, 2243, 2312 ], "unit_up_proj_g22": [ 2138, 2146, 2237, 2287, 2316, 2432, 2876, 2986 ], "unit_down_proj_g22": [ 1851, 1874, 1900, 1917, 2027, 2106, 2412, 2549, 2698, 3173 ], "unit_down_proj_g30": [ 1856, 2076, 2229 ], "unit_down_proj_g24": [ 1836, 1852, 2162, 2201, 2325, 2487, 2535 ], "unit_up_proj_g06": [ 2190, 2227, 2679, 2685, 2702, 2733, 2948, 3035, 3204, 3291 ], "unit_v_proj_g16": [ 1646, 1649, 1677, 1839, 1916, 2007, 2049, 2055, 2090, 2143, 2168, 2279, 2403, 2485, 2522, 2780, 2949, 3114, 3329, 3448, 3734 ], "unit_down_proj_g03": [ 1851, 2192, 2296 ], "unit_gate_proj_g30": [ 2078, 2095, 2104, 2317, 3279, 3290 ], "unit_up_proj_g28": [ 2336 ], "unit_gate_proj_g10": [ 2227, 2561, 2603, 2950, 3127 ], "unit_o_proj_g04": [ 1259, 1323, 1594, 1727 ], "unit_up_proj_g04": [ 2761 ], "unit_q_proj_g12": [ 2319, 2396 ], "unit_down_proj_g00": [ 1827, 2315 ], "unit_down_proj_g02": [ 2179 ], "unit_k_proj_g26": [ 1739 ], "unit_gate_proj_g20": [ 2165, 2193, 2386, 2397, 2426, 2817 ], "unit_up_proj_g30": [ 2225, 2240, 2277, 2881, 3648 ], "unit_k_proj_g22": [ 2091 ], "unit_o_proj_g22": [ 1353, 1401, 1896 ], "unit_gate_proj_g16": [ 2084, 2169, 2400, 2414, 2467, 2562, 3064, 3504, 3517, 4008 ], "unit_o_proj_g19": [ 1267, 1302, 1316, 1335, 1417, 1558, 1651, 1666, 1667, 1890, 2014, 2042, 2063 ], "unit_o_proj_g23": [ 1273, 1459, 1483, 2068, 2259 ], "unit_o_proj_g27": [ 1229, 1361, 1421 ], "unit_gate_proj_g04": [ 2357, 2878, 3101 ], "unit_o_proj_g31": [ 1255, 1257, 1378, 1501 ], "unit_gate_proj_g02": [ 2292, 4051 ], "unit_up_proj_g26": [ 2393, 2608, 2751, 2794 ], "unit_gate_proj_g26": [ 2293 ], "unit_k_proj_g30": [ 1752, 2028, 2356 ], "unit_down_proj_g23": [ 2401, 2414 ], "unit_q_proj_g14": [ 1791 ], "unit_k_proj_g18": [ 2030, 2933 ], "unit_q_proj_g30": [ 2219 ], "unit_k_proj_g14": [ 1731 ], "unit_o_proj_g05": [ 1280, 1369, 1417, 1446, 1594, 1749, 1752 ], "unit_q_proj_g20": [ 1647 ], "unit_k_proj_g06": [ 1780 ], "unit_q_proj_g18": [ 1777 ], "unit_q_proj_g08": [ 1734 ], "unit_k_proj_g28": [ 2675 ] }, "evicted": { "unit_gate_proj_g28": [ 2065, 2066, 2067, 2068, 2069, 2070, 2071 ], "unit_o_proj_g09": [ 1210, 1211, 1212, 1213, 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227 ], "unit_o_proj_g15": [ 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227 ], "unit_o_proj_g10": [ 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227 ], "unit_v_proj_g30": [ 1629, 1630, 1631, 1632, 1633, 1634, 1635, 1636, 1637 ], "unit_v_proj_g10": [ 1612, 1613, 1614, 1615, 1616, 1617, 1618, 1619, 1620, 1621, 1622, 1623, 1624, 1625, 1626, 1627, 1628, 1629, 1630, 1631, 1632, 1633, 1634, 1635, 1636, 1637 ], "unit_o_proj_g21": [ 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227 ], "unit_o_proj_g06": [ 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227 ], "unit_v_proj_g08": [ 1634, 1635, 1636, 1637 ], "unit_down_proj_g15": [ 1771, 1772, 1773, 1774, 1775, 1776, 1777, 1778, 1779, 1780, 1781, 1782, 1783, 1784, 1785, 1786, 1787, 1788, 1789, 1790 ], "unit_down_proj_g13": [ 1786, 1787, 1788, 1789, 1790 ], "unit_down_proj_g09": [ 1772, 1773, 1774, 1775, 1776, 1777, 1778, 1779, 1780, 1781, 1782, 1783, 1784, 1785, 1786, 1787, 1788, 1789, 1790 ], "unit_down_proj_g05": [ 1774, 1775, 1776, 1777, 1778, 1779, 1780, 1781, 1782, 1783, 1784, 1785, 1786, 1787, 1788, 1789, 1790 ], "unit_v_proj_g14": [ 1636, 1637 ], "unit_o_proj_g28": [ 1208, 1209, 1210, 1211, 1212, 1213, 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227 ], "unit_o_proj_g08": [ 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211, 1212, 1213, 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227 ], "unit_o_proj_g20": [ 1223, 1224, 1225, 1226, 1227 ], "unit_v_proj_g04": [ 1608, 1609, 1610, 1611, 1612, 1613, 1614, 1615, 1616, 1617, 1618, 1619, 1620, 1621, 1622, 1623, 1624, 1625, 1626, 1627, 1628, 1629, 1630, 1631, 1632, 1633, 1634, 1635, 1636, 1637 ], "unit_v_proj_g12": [ 1621, 1622, 1623, 1624, 1625, 1626, 1627, 1628, 1629, 1630, 1631, 1632, 1633, 1634, 1635, 1636, 1637 ], "unit_o_proj_g29": [ 1226, 1227 ], "unit_v_proj_g28": [ 1630, 1631, 1632, 1633, 1634, 1635, 1636, 1637 ], "unit_v_proj_g20": [ 1632, 1633, 1634, 1635, 1636, 1637 ], "unit_v_proj_g18": [ 1630, 1631, 1632, 1633, 1634, 1635, 1636, 1637 ], "unit_down_proj_g07": [ 1784, 1785, 1786, 1787, 1788, 1789, 1790 ], "unit_o_proj_g01": [ 1227 ], "unit_o_proj_g11": [ 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227 ], "unit_v_proj_g06": [ 1620, 1621, 1622, 1623, 1624, 1625, 1626, 1627, 1628, 1629, 1630, 1631, 1632, 1633, 1634, 1635, 1636, 1637 ], "unit_up_proj_g14": [ 2058, 2059, 2060, 2061, 2062, 2063, 2064, 2065, 2066, 2067, 2068, 2069, 2070, 2071 ], "unit_down_proj_g06": [ 1789, 1790 ], "unit_o_proj_g16": [ 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227 ], "unit_down_proj_g12": [ 1772, 1773, 1774, 1775, 1776, 1777, 1778, 1779, 1780, 1781, 1782, 1783, 1784, 1785, 1786, 1787, 1788, 1789, 1790 ], "unit_o_proj_g03": [ 1218, 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227 ], "unit_v_proj_g24": [ 1633, 1634, 1635, 1636, 1637 ], "unit_down_proj_g17": [ 1770, 1771, 1772, 1773, 1774, 1775, 1776, 1777, 1778, 1779, 1780, 1781, 1782, 1783, 1784, 1785, 1786, 1787, 1788, 1789, 1790 ], "unit_down_proj_g18": [ 1780, 1781, 1782, 1783, 1784, 1785, 1786, 1787, 1788, 1789, 1790 ], "unit_o_proj_g25": [ 1223, 1224, 1225, 1226, 1227 ], "unit_down_proj_g16": [ 1782, 1783, 1784, 1785, 1786, 1787, 1788, 1789, 1790 ], "unit_o_proj_g13": [ 1224, 1225, 1226, 1227 ], "unit_up_proj_g08": [ 2071 ], "unit_down_proj_g10": [ 1787, 1788, 1789, 1790 ], "unit_down_proj_g11": [ 1782, 1783, 1784, 1785, 1786, 1787, 1788, 1789, 1790 ], "unit_gate_proj_g12": [ 2068, 2069, 2070, 2071 ], "unit_down_proj_g08": [ 1786, 1787, 1788, 1789, 1790 ], "unit_o_proj_g14": [ 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227 ], "unit_up_proj_g16": [ 2071 ], "unit_down_proj_g14": [ 1788, 1789, 1790 ], "unit_q_proj_g16": [ 1633, 1634, 1635, 1636, 1637 ], "unit_v_proj_g02": [ 1624, 1625, 1626, 1627, 1628, 1629, 1630, 1631, 1632, 1633, 1634, 1635, 1636, 1637 ], "unit_up_proj_g20": [ 2066, 2067, 2068, 2069, 2070, 2071 ], "unit_down_proj_g19": [ 1782, 1783, 1784, 1785, 1786, 1787, 1788, 1789, 1790 ], "unit_down_proj_g21": [ 1785, 1786, 1787, 1788, 1789, 1790 ], "unit_down_proj_g20": [ 1779, 1780, 1781, 1782, 1783, 1784, 1785, 1786, 1787, 1788, 1789, 1790 ], "unit_down_proj_g01": [ 1788, 1789, 1790 ], "unit_v_proj_g26": [ 1618, 1619, 1620, 1621, 1622, 1623, 1624, 1625, 1626, 1627, 1628, 1629, 1630, 1631, 1632, 1633, 1634, 1635, 1636, 1637 ], "unit_o_proj_g18": [ 1211, 1212, 1213, 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227 ], "unit_gate_proj_g14": [ 2061, 2062, 2063, 2064, 2065, 2066, 2067, 2068, 2069, 2070, 2071 ], "unit_o_proj_g30": [ 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227 ], "unit_up_proj_g10": [ 2060, 2061, 2062, 2063, 2064, 2065, 2066, 2067, 2068, 2069, 2070, 2071 ], "unit_down_proj_g26": [ 1787, 1788, 1789, 1790 ], "unit_o_proj_g07": [ 1199, 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211, 1212, 1213, 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227 ], "unit_down_proj_g31": [ 1790 ], "unit_down_proj_g04": [ 1770, 1771, 1772, 1773, 1774, 1775, 1776, 1777, 1778, 1779, 1780, 1781, 1782, 1783, 1784, 1785, 1786, 1787, 1788, 1789, 1790 ], "unit_v_proj_g22": [ 1632, 1633, 1634, 1635, 1636, 1637 ], "unit_up_proj_g02": [ 2069, 2070, 2071 ], "unit_gate_proj_g08": [ 2065, 2066, 2067, 2068, 2069, 2070, 2071 ], "unit_o_proj_g24": [ 1227 ], "unit_down_proj_g29": [ 1783, 1784, 1785, 1786, 1787, 1788, 1789, 1790 ], "unit_up_proj_g12": [ 2068, 2069, 2070, 2071 ], "unit_up_proj_g18": [ 2061, 2062, 2063, 2064, 2065, 2066, 2067, 2068, 2069, 2070, 2071 ], "unit_down_proj_g27": [ 1789, 1790 ], "unit_down_proj_g28": [ 1788, 1789, 1790 ], "unit_o_proj_g02": [ 1221, 1222, 1223, 1224, 1225, 1226, 1227 ], "unit_up_proj_g24": [ 2071 ], "unit_o_proj_g17": [ 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227 ], "unit_gate_proj_g06": [ 2066, 2067, 2068, 2069, 2070, 2071 ], "unit_q_proj_g06": [ 1637 ], "unit_o_proj_g26": [ 1222, 1223, 1224, 1225, 1226, 1227 ], "unit_gate_proj_g18": [ 2069, 2070, 2071 ], "unit_down_proj_g25": [ 1786, 1787, 1788, 1789, 1790 ], "unit_up_proj_g22": [ 2064, 2065, 2066, 2067, 2068, 2069, 2070, 2071 ], "unit_down_proj_g22": [ 1781, 1782, 1783, 1784, 1785, 1786, 1787, 1788, 1789, 1790 ], "unit_down_proj_g30": [ 1788, 1789, 1790 ], "unit_down_proj_g24": [ 1784, 1785, 1786, 1787, 1788, 1789, 1790 ], "unit_up_proj_g06": [ 2062, 2063, 2064, 2065, 2066, 2067, 2068, 2069, 2070, 2071 ], "unit_v_proj_g16": [ 1617, 1618, 1619, 1620, 1621, 1622, 1623, 1624, 1625, 1626, 1627, 1628, 1629, 1630, 1631, 1632, 1633, 1634, 1635, 1636, 1637 ], "unit_down_proj_g03": [ 1788, 1789, 1790 ], "unit_gate_proj_g30": [ 2066, 2067, 2068, 2069, 2070, 2071 ], "unit_up_proj_g28": [ 2071 ], "unit_gate_proj_g10": [ 2067, 2068, 2069, 2070, 2071 ], "unit_o_proj_g04": [ 1224, 1225, 1226, 1227 ], "unit_up_proj_g04": [ 2071 ], "unit_q_proj_g12": [ 1636, 1637 ], "unit_down_proj_g00": [ 1789, 1790 ], "unit_down_proj_g02": [ 1790 ], "unit_k_proj_g26": [ 1637 ], "unit_gate_proj_g20": [ 2066, 2067, 2068, 2069, 2070, 2071 ], "unit_up_proj_g30": [ 2067, 2068, 2069, 2070, 2071 ], "unit_k_proj_g22": [ 1637 ], "unit_o_proj_g22": [ 1225, 1226, 1227 ], "unit_gate_proj_g16": [ 2062, 2063, 2064, 2065, 2066, 2067, 2068, 2069, 2070, 2071 ], "unit_o_proj_g19": [ 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227 ], "unit_o_proj_g23": [ 1223, 1224, 1225, 1226, 1227 ], "unit_o_proj_g27": [ 1225, 1226, 1227 ], "unit_gate_proj_g04": [ 2069, 2070, 2071 ], "unit_o_proj_g31": [ 1224, 1225, 1226, 1227 ], "unit_gate_proj_g02": [ 2070, 2071 ], "unit_up_proj_g26": [ 2068, 2069, 2070, 2071 ], "unit_gate_proj_g26": [ 2071 ], "unit_k_proj_g30": [ 1635, 1636, 1637 ], "unit_down_proj_g23": [ 1789, 1790 ], "unit_q_proj_g14": [ 1637 ], "unit_k_proj_g18": [ 1636, 1637 ], "unit_q_proj_g30": [ 1637 ], "unit_k_proj_g14": [ 1637 ], "unit_o_proj_g05": [ 1221, 1222, 1223, 1224, 1225, 1226, 1227 ], "unit_q_proj_g20": [ 1637 ], "unit_k_proj_g06": [ 1637 ], "unit_q_proj_g18": [ 1637 ], "unit_q_proj_g08": [ 1637 ], "unit_k_proj_g28": [ 1637 ] }, "stopped": "snapshot after round 2", "predicted_orr_total": -0.32461821016369186, "history": [ { "round": 0, "n_added": 0, "spent": 0, "measured": { "refusal_nll": 2.85142471075058, "answer_nll": 0.8801046538352968, "direct_loss": 3.7315293645858767, "dsafe": 2.090403609275818, "dsafe_sem": 0.03944625784696569, "orr": -2.423324728012085, "orr_sem": 0.03990019692182367, "refh": -2.85142471075058, "refh_sem": 0.029137263047875794, "refb": -3.3034293818473817, "refb_sem": 0.021596701807731964, "gap": -0.45200467109680176, "gap_sem": 0.0362684108680433 } }, { "round": 1, "n_added": 454, "n_swapped_out": 454, "added": { "unit_gate_proj_g28": [ 4085, 2433, 2437 ], "unit_o_proj_g09": [ 1286, 1659, 1288, 1667, 1503, 2329, 1301, 1294 ], "unit_o_proj_g15": [ 2394, 1577, 1284, 1765, 2462, 1286, 2137, 1527 ], "unit_o_proj_g10": [ 1301, 1245, 1369, 2342, 1381, 1610, 1243, 1768, 1992, 1461, 1889, 2173, 1264 ], "unit_v_proj_g30": [ 2112, 2601, 2301 ], "unit_v_proj_g10": [ 2266, 1785, 2802, 2345, 2778, 2609, 1844, 1981 ], "unit_o_proj_g21": [ 1917, 2415, 1911, 2119, 1377 ], "unit_o_proj_g06": [ 1933, 1359, 1229, 1445, 1479, 1426, 1463, 2498, 1701, 1719, 1282 ], "unit_v_proj_g08": [ 1698, 2147, 2736, 2982 ], "unit_down_proj_g15": [ 2735, 2302, 2461, 2203 ], "unit_down_proj_g13": [ 2211, 3341, 2004, 1857, 2652 ], "unit_down_proj_g09": [ 2044, 2380, 2013, 2018, 1817, 1800, 2346, 2931, 2305, 1880, 2802, 2504, 1975, 2916, 3524, 2108, 2518, 3316 ], "unit_down_proj_g05": [ 2150, 1809, 1849, 2691, 2800, 1832, 2423, 2009 ], "unit_v_proj_g14": [ 1796, 1932 ], "unit_o_proj_g28": [ 2966, 1763, 1649, 1273, 1307, 1459, 1736, 1985, 1605 ], "unit_o_proj_g08": [ 1228, 1736, 1385, 1279, 1848, 1377, 1797, 2048, 1237 ], "unit_o_proj_g20": [ 1516, 1357, 1693 ], "unit_v_proj_g04": [ 3006, 2975, 3351, 2960 ], "unit_v_proj_g12": [ 2670, 1843, 2193, 3487, 2314, 1831, 1945, 2230, 2842, 1830, 1898, 2574, 2059, 2970, 2632 ], "unit_o_proj_g29": [ 1275 ], "unit_v_proj_g28": [ 4091, 1657, 1815, 2903 ], "unit_v_proj_g20": [ 2513, 2195, 1801, 1923, 2270, 2688 ], "unit_v_proj_g18": [ 2435, 2252, 2431 ], "unit_down_proj_g07": [ 2110, 1858, 2591, 2057, 1884, 1967, 2048 ], "unit_o_proj_g01": [ 1253 ], "unit_o_proj_g11": [ 1347, 1460, 1258, 1333, 1571, 1580 ], "unit_v_proj_g06": [ 2567, 2694, 3264, 1749, 2946, 3527, 2893, 2457, 1942, 1712, 1687, 3157, 2423, 3665 ], "unit_up_proj_g14": [ 2243, 3909, 2544, 2339, 2829, 2231, 2539, 4074 ], "unit_down_proj_g06": [ 2004, 2002 ], "unit_o_proj_g16": [ 1658, 1458 ], "unit_down_proj_g12": [ 2046, 2534, 2722, 1909, 2228, 2430, 2336, 2091, 2025, 1976 ], "unit_o_proj_g03": [ 1276, 2525, 1408, 1264, 1501, 1976, 1243 ], "unit_v_proj_g24": [ 1882, 1913 ], "unit_down_proj_g17": [ 2379, 1910, 2605, 2992, 3188, 2248, 2470, 2182, 2667, 2611, 2592, 2209, 2086 ], "unit_down_proj_g18": [ 1861, 1813, 2582, 2422, 2397 ], "unit_o_proj_g25": [ 2107, 1283 ], "unit_down_proj_g16": [ 2119, 2377, 1903, 2759, 1857, 2213, 2182 ], "unit_o_proj_g13": [ 1453, 1644, 2079, 1466 ], "unit_up_proj_g08": [ 2906 ], "unit_down_proj_g10": [ 1981, 1909 ], "unit_down_proj_g11": [ 2031, 2216, 1930 ], "unit_gate_proj_g12": [ 2268, 2280 ], "unit_down_proj_g08": [ 2473, 1960, 2200, 1953, 1805 ], "unit_o_proj_g14": [ 2004, 1287, 2257, 1230, 1264, 1460, 1657, 1367, 1280, 1331 ], "unit_up_proj_g16": [ 3002 ], "unit_down_proj_g14": [ 1928, 1863, 1826 ], "unit_q_proj_g16": [ 2215, 1655, 3080 ], "unit_v_proj_g02": [ 3996, 4021 ], "unit_up_proj_g20": [ 2572, 2213, 2241, 3102, 3787 ], "unit_down_proj_g19": [ 1843, 1858, 1903, 2490, 1936, 2864, 1950, 2893, 2255 ], "unit_down_proj_g21": [ 3423, 2104, 2166, 2042, 3396, 1918 ], "unit_down_proj_g20": [ 2345, 1958, 1971, 2623, 2011, 2782, 1893, 1862, 2489 ], "unit_down_proj_g01": [ 1863, 1822, 1874 ], "unit_v_proj_g26": [ 1784, 1795, 3009, 1766, 2513, 2501 ], "unit_o_proj_g18": [ 1455, 1277, 2014, 1411, 1492, 1582 ], "unit_gate_proj_g14": [ 2705, 2584 ], "unit_o_proj_g30": [ 1245, 1240 ], "unit_up_proj_g10": [ 3603, 2338, 2666, 3219, 2525, 3565, 2287, 4047 ], "unit_down_proj_g26": [ 3048 ], "unit_o_proj_g07": [ 1460, 2038, 1331, 2423, 1786, 2194, 2361, 1421, 1785, 2801, 1561, 1758, 1393 ], "unit_down_proj_g31": [ 2278 ], "unit_down_proj_g04": [ 1911, 1843, 2453, 2026, 2666 ], "unit_v_proj_g22": [ 2316, 2035, 2715 ], "unit_up_proj_g02": [ 2401, 2197 ], "unit_gate_proj_g08": [ 2152 ], "unit_o_proj_g24": [ 1384 ], "unit_down_proj_g29": [ 1874, 2339, 1858, 2333, 1833 ], "unit_up_proj_g12": [ 2329, 3744, 2617, 3057 ], "unit_up_proj_g18": [ 3401, 2779 ], "unit_down_proj_g27": [ 2104 ], "unit_down_proj_g28": [ 3146 ], "unit_o_proj_g02": [ 1273 ], "unit_up_proj_g24": [ 2371 ], "unit_o_proj_g17": [ 1608, 1901, 1233 ], "unit_gate_proj_g06": [ 2456, 2706, 4081, 2837, 2531, 3053 ], "unit_q_proj_g06": [ 1772 ], "unit_o_proj_g26": [ 1774, 2459, 1643, 1535, 1523, 2886 ], "unit_gate_proj_g18": [ 2429, 2624, 2321 ], "unit_down_proj_g25": [ 2312, 2243 ], "unit_up_proj_g22": [ 2876, 2146, 2138 ], "unit_down_proj_g22": [ 1874, 2027 ], "unit_down_proj_g30": [ 1856 ], "unit_down_proj_g24": [ 2487, 2535, 2162, 1836, 2325 ], "unit_up_proj_g06": [ 2702, 2679, 2948, 3204, 2733, 3291 ], "unit_v_proj_g16": [ 3114, 2279, 2168, 1839, 1649, 2049, 2522, 3734, 2143, 2780, 1646 ], "unit_down_proj_g03": [ 2296, 1851, 2192 ], "unit_gate_proj_g30": [ 2095, 2078 ], "unit_up_proj_g28": [ 2336 ], "unit_gate_proj_g10": [ 2950, 3127, 2603 ], "unit_o_proj_g04": [ 1727, 1259, 1323, 1594 ], "unit_up_proj_g04": [ 2761 ], "unit_q_proj_g12": [ 2319, 2396 ], "unit_down_proj_g00": [ 2315 ], "unit_down_proj_g02": [ 2179 ], "unit_k_proj_g26": [ 1739 ], "unit_gate_proj_g20": [ 2193 ], "unit_up_proj_g30": [ 2225, 2240 ], "unit_k_proj_g22": [ 2091 ], "unit_o_proj_g22": [ 1401 ], "unit_gate_proj_g16": [ 3504, 2400, 2084, 2467, 3064, 3517, 2169 ] }, "dropped": { "unit_gate_proj_g28": [ 2069, 2070, 2071 ], "unit_o_proj_g09": [ 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227 ], "unit_o_proj_g15": [ 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227 ], "unit_o_proj_g10": [ 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227 ], "unit_v_proj_g30": [ 1635, 1636, 1637 ], "unit_v_proj_g10": [ 1630, 1631, 1632, 1633, 1634, 1635, 1636, 1637 ], "unit_o_proj_g21": [ 1223, 1224, 1225, 1226, 1227 ], "unit_o_proj_g06": [ 1217, 1218, 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227 ], "unit_v_proj_g08": [ 1634, 1635, 1636, 1637 ], "unit_down_proj_g15": [ 1787, 1788, 1789, 1790 ], "unit_down_proj_g13": [ 1786, 1787, 1788, 1789, 1790 ], "unit_down_proj_g09": [ 1773, 1774, 1775, 1776, 1777, 1778, 1779, 1780, 1781, 1782, 1783, 1784, 1785, 1786, 1787, 1788, 1789, 1790 ], "unit_down_proj_g05": [ 1783, 1784, 1785, 1786, 1787, 1788, 1789, 1790 ], "unit_v_proj_g14": [ 1636, 1637 ], "unit_o_proj_g28": [ 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227 ], "unit_o_proj_g08": [ 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227 ], "unit_o_proj_g20": [ 1225, 1226, 1227 ], "unit_v_proj_g04": [ 1634, 1635, 1636, 1637 ], "unit_v_proj_g12": [ 1623, 1624, 1625, 1626, 1627, 1628, 1629, 1630, 1631, 1632, 1633, 1634, 1635, 1636, 1637 ], "unit_o_proj_g29": [ 1227 ], "unit_v_proj_g28": [ 1634, 1635, 1636, 1637 ], "unit_v_proj_g20": [ 1632, 1633, 1634, 1635, 1636, 1637 ], "unit_v_proj_g18": [ 1635, 1636, 1637 ], "unit_down_proj_g07": [ 1784, 1785, 1786, 1787, 1788, 1789, 1790 ], "unit_o_proj_g01": [ 1227 ], "unit_o_proj_g11": [ 1222, 1223, 1224, 1225, 1226, 1227 ], "unit_v_proj_g06": [ 1624, 1625, 1626, 1627, 1628, 1629, 1630, 1631, 1632, 1633, 1634, 1635, 1636, 1637 ], "unit_up_proj_g14": [ 2064, 2065, 2066, 2067, 2068, 2069, 2070, 2071 ], "unit_down_proj_g06": [ 1789, 1790 ], "unit_o_proj_g16": [ 1226, 1227 ], "unit_down_proj_g12": [ 1781, 1782, 1783, 1784, 1785, 1786, 1787, 1788, 1789, 1790 ], "unit_o_proj_g03": [ 1221, 1222, 1223, 1224, 1225, 1226, 1227 ], "unit_v_proj_g24": [ 1636, 1637 ], "unit_down_proj_g17": [ 1778, 1779, 1780, 1781, 1782, 1783, 1784, 1785, 1786, 1787, 1788, 1789, 1790 ], "unit_down_proj_g18": [ 1786, 1787, 1788, 1789, 1790 ], "unit_o_proj_g25": [ 1226, 1227 ], "unit_down_proj_g16": [ 1784, 1785, 1786, 1787, 1788, 1789, 1790 ], "unit_o_proj_g13": [ 1224, 1225, 1226, 1227 ], "unit_up_proj_g08": [ 2071 ], "unit_down_proj_g10": [ 1789, 1790 ], "unit_down_proj_g11": [ 1788, 1789, 1790 ], "unit_gate_proj_g12": [ 2070, 2071 ], "unit_down_proj_g08": [ 1786, 1787, 1788, 1789, 1790 ], "unit_o_proj_g14": [ 1218, 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227 ], "unit_up_proj_g16": [ 2071 ], "unit_down_proj_g14": [ 1788, 1789, 1790 ], "unit_q_proj_g16": [ 1635, 1636, 1637 ], "unit_v_proj_g02": [ 1636, 1637 ], "unit_up_proj_g20": [ 2067, 2068, 2069, 2070, 2071 ], "unit_down_proj_g19": [ 1782, 1783, 1784, 1785, 1786, 1787, 1788, 1789, 1790 ], "unit_down_proj_g21": [ 1785, 1786, 1787, 1788, 1789, 1790 ], "unit_down_proj_g20": [ 1782, 1783, 1784, 1785, 1786, 1787, 1788, 1789, 1790 ], "unit_down_proj_g01": [ 1788, 1789, 1790 ], "unit_v_proj_g26": [ 1632, 1633, 1634, 1635, 1636, 1637 ], "unit_o_proj_g18": [ 1222, 1223, 1224, 1225, 1226, 1227 ], "unit_gate_proj_g14": [ 2070, 2071 ], "unit_o_proj_g30": [ 1226, 1227 ], "unit_up_proj_g10": [ 2064, 2065, 2066, 2067, 2068, 2069, 2070, 2071 ], "unit_down_proj_g26": [ 1790 ], "unit_o_proj_g07": [ 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227 ], "unit_down_proj_g31": [ 1790 ], "unit_down_proj_g04": [ 1786, 1787, 1788, 1789, 1790 ], "unit_v_proj_g22": [ 1635, 1636, 1637 ], "unit_up_proj_g02": [ 2070, 2071 ], "unit_gate_proj_g08": [ 2071 ], "unit_o_proj_g24": [ 1227 ], "unit_down_proj_g29": [ 1786, 1787, 1788, 1789, 1790 ], "unit_up_proj_g12": [ 2068, 2069, 2070, 2071 ], "unit_up_proj_g18": [ 2070, 2071 ], "unit_down_proj_g27": [ 1790 ], "unit_down_proj_g28": [ 1790 ], "unit_o_proj_g02": [ 1227 ], "unit_up_proj_g24": [ 2071 ], "unit_o_proj_g17": [ 1225, 1226, 1227 ], "unit_gate_proj_g06": [ 2066, 2067, 2068, 2069, 2070, 2071 ], "unit_q_proj_g06": [ 1637 ], "unit_o_proj_g26": [ 1222, 1223, 1224, 1225, 1226, 1227 ], "unit_gate_proj_g18": [ 2069, 2070, 2071 ], "unit_down_proj_g25": [ 1789, 1790 ], "unit_up_proj_g22": [ 2069, 2070, 2071 ], "unit_down_proj_g22": [ 1789, 1790 ], "unit_down_proj_g30": [ 1790 ], "unit_down_proj_g24": [ 1786, 1787, 1788, 1789, 1790 ], "unit_up_proj_g06": [ 2066, 2067, 2068, 2069, 2070, 2071 ], "unit_v_proj_g16": [ 1627, 1628, 1629, 1630, 1631, 1632, 1633, 1634, 1635, 1636, 1637 ], "unit_down_proj_g03": [ 1788, 1789, 1790 ], "unit_gate_proj_g30": [ 2070, 2071 ], "unit_up_proj_g28": [ 2071 ], "unit_gate_proj_g10": [ 2069, 2070, 2071 ], "unit_o_proj_g04": [ 1224, 1225, 1226, 1227 ], "unit_up_proj_g04": [ 2071 ], "unit_q_proj_g12": [ 1636, 1637 ], "unit_down_proj_g00": [ 1790 ], "unit_down_proj_g02": [ 1790 ], "unit_k_proj_g26": [ 1637 ], "unit_gate_proj_g20": [ 2071 ], "unit_up_proj_g30": [ 2070, 2071 ], "unit_k_proj_g22": [ 1637 ], "unit_o_proj_g22": [ 1227 ], "unit_gate_proj_g16": [ 2065, 2066, 2067, 2068, 2069, 2070, 2071 ] }, "added_by_type": { "gate_proj": 30, "o_proj": 136, "v_proj": 87, "down_proj": 148, "up_proj": 45, "q_proj": 6, "k_proj": 2 }, "spent": 6475264, "param_fraction": 0.5999091746275906, "predicted_orr": -0.16642059284350808, "predicted_orr_total": -0.16642059284350808, "tau_binding": false, "measured": { "refusal_nll": 2.6448221015930176, "answer_nll": 0.8331169700622558, "direct_loss": 3.4779390716552734, "dsafe": 1.6802209401130677, "dsafe_sem": 0.0507437811338878, "orr": -2.5466210293769835, "orr_sem": 0.04562753735166352, "refh": -2.6448221015930176, "refh_sem": 0.033102309822432154, "refb": -3.3797379994392394, "refb_sem": 0.02906218676584838, "gap": -0.7349158978462218, "gap_sem": 0.04404967213491311 }, "predicted_insert_value": 0.48447243668488227, "predicted_removal_value": 0.034292311895114835, "insert_cancellation": { "sum_abs_layer_value": 0.48615083465119824, "abs_sum_value": 0.4844724353752099, "n_opposite_sign": 5 }, "evict_cancellation": { "sum_abs_layer_value": 0.13125207840494113, "abs_sum_value": 0.11347390184528194, "n_opposite_sign": 83 }, "seconds": 88.01406693458557 }, { "round": 2, "n_added": 470, "n_swapped_out": 470, "added": { "unit_o_proj_g09": [ 1924, 1717, 1367, 1541, 1376, 2196, 1733, 2904, 2991, 1642 ], "unit_v_proj_g30": [ 4095, 1753, 2109, 2451, 3340, 3135 ], "unit_o_proj_g14": [ 1483, 1478, 1663, 1270 ], "unit_down_proj_g18": [ 3400, 2011, 2261, 3574, 1849, 2182 ], "unit_o_proj_g19": [ 1335, 1651, 1316, 1666, 1417, 1302, 1890, 2014, 1267, 1558, 1667, 2063, 2042 ], "unit_down_proj_g17": [ 2718, 2111, 1920, 3349, 1855, 2925, 2777, 2939 ], "unit_o_proj_g20": [ 1440, 1674 ], "unit_o_proj_g23": [ 1273, 1483, 1459, 2259, 2068 ], "unit_o_proj_g08": [ 1335, 1373, 1505, 1230, 1758, 1287, 2138, 1381, 1804, 1922, 1378, 1584, 1466, 2257, 1721 ], "unit_o_proj_g18": [ 1970, 1342, 1769, 1329, 1948, 1897, 2010, 1458, 1322, 1763, 1290 ], "unit_o_proj_g21": [ 1274, 2080, 1372, 1410, 1297, 1918, 2042 ], "unit_o_proj_g27": [ 1229, 1421, 1361 ], "unit_v_proj_g26": [ 2125, 2181, 2558, 2103, 1825, 2346, 2194, 2036, 1972, 1698, 1831, 2011, 3512, 2965 ], "unit_down_proj_g10": [ 1796, 1989 ], "unit_down_proj_g12": [ 2674, 1858, 1999, 2117, 2683, 2219, 2671, 2436, 2752 ], "unit_v_proj_g06": [ 1845, 2233, 2501, 3110 ], "unit_gate_proj_g14": [ 2106, 3396, 2565, 3072, 2769, 2538, 3503, 2128, 2564 ], "unit_gate_proj_g04": [ 3101, 2878, 2357 ], "unit_v_proj_g02": [ 3487, 4016, 2237, 3643, 2255, 4071, 1750, 3817, 4019, 2754, 3529, 3048 ], "unit_gate_proj_g20": [ 2386, 2397, 2817, 2165, 2426 ], "unit_up_proj_g14": [ 2126, 2273, 2996, 3258, 3047, 2534 ], "unit_o_proj_g17": [ 1571, 1498, 1492, 2226, 1486, 1464, 1426, 1524, 1776, 1528 ], "unit_down_proj_g28": [ 1828, 1808 ], "unit_down_proj_g04": [ 2044, 2758, 1794, 1957, 2031, 2005, 1999, 2597, 2173, 2202, 2233, 2231, 2129, 2092, 2142, 2583 ], "unit_o_proj_g22": [ 1896, 1353 ], "unit_v_proj_g28": [ 3557, 2627, 2255, 1845 ], "unit_v_proj_g18": [ 2116, 1688, 1992, 2022, 1650 ], "unit_up_proj_g10": [ 2646, 2090, 3355, 2236 ], "unit_down_proj_g25": [ 2006, 1995, 1908 ], "unit_v_proj_g12": [ 1751, 1750 ], "unit_o_proj_g06": [ 1534, 1389, 2048 ], "unit_down_proj_g26": [ 2070, 2366, 2190 ], "unit_down_proj_g11": [ 2091, 1809, 2005, 2463, 2687, 2767 ], "unit_gate_proj_g12": [ 2601, 2254 ], "unit_o_proj_g31": [ 1501, 1378, 1255, 1257 ], "unit_gate_proj_g02": [ 2292, 4051 ], "unit_gate_proj_g30": [ 3290, 3279, 2104, 2317 ], "unit_gate_proj_g08": [ 2693, 2896, 2221, 2300, 2458, 2620 ], "unit_o_proj_g03": [ 1609, 1561, 1250 ], "unit_down_proj_g05": [ 2593, 1861, 1893, 2225, 2133, 1945, 2195, 2956, 3153 ], "unit_o_proj_g29": [ 1322 ], "unit_v_proj_g10": [ 1973, 2231, 2196, 2402, 1994, 2762, 1890, 2575, 2457, 1841, 1784, 1670, 3694, 1745, 1957, 2047, 2994, 1888 ], "unit_o_proj_g16": [ 1383, 1413, 1802, 1330, 1472, 1504, 1741, 1267, 1276, 2411, 1302 ], "unit_down_proj_g22": [ 2412, 1917, 1851, 2698, 1900, 2549, 3173, 2106 ], "unit_up_proj_g26": [ 2794, 2608, 2751, 2393 ], "unit_gate_proj_g26": [ 2293 ], "unit_o_proj_g28": [ 1504, 2146, 1319, 1981, 1755, 1631, 1873, 1277, 1538, 1270, 1600 ], "unit_o_proj_g15": [ 1636, 1395, 1826, 1497, 1516 ], "unit_o_proj_g07": [ 1244, 1293, 1940, 1485, 1443, 1372, 1304, 1234, 1350, 2326, 1489, 1753, 2610, 1285, 1780, 1601 ], "unit_o_proj_g25": [ 1313, 1482, 1308 ], "unit_down_proj_g30": [ 2229, 2076 ], "unit_gate_proj_g10": [ 2561, 2227 ], "unit_down_proj_g00": [ 1827 ], "unit_up_proj_g18": [ 2078, 2272, 2571, 2705, 2135, 2480, 2459, 2141, 2605 ], "unit_down_proj_g24": [ 2201, 1852 ], "unit_down_proj_g27": [ 1813 ], "unit_k_proj_g30": [ 2028, 2356, 1752 ], "unit_up_proj_g22": [ 2237, 2316, 2432, 2287, 2986 ], "unit_down_proj_g29": [ 2009, 2308, 2087 ], "unit_up_proj_g02": [ 2141 ], "unit_up_proj_g20": [ 2376 ], "unit_down_proj_g23": [ 2414, 2401 ], "unit_up_proj_g30": [ 2881, 3648, 2277 ], "unit_o_proj_g11": [ 1486, 1465, 1244, 1520, 1249, 1521, 2013 ], "unit_v_proj_g22": [ 2077, 2716, 2340 ], "unit_q_proj_g14": [ 1791 ], "unit_k_proj_g18": [ 2030, 2933 ], "unit_gate_proj_g16": [ 4008, 2414, 2562 ], "unit_down_proj_g15": [ 2434, 2825, 2198, 2945, 1928, 3124, 2207, 1806, 2661, 2890, 2277, 2092, 2079, 3736, 2594, 2608 ], "unit_q_proj_g30": [ 2219 ], "unit_v_proj_g04": [ 2312, 3122, 2593, 2622, 2820, 2237, 1739, 2213, 2359, 2557, 3138, 3147, 3324, 3406, 2056, 1678, 2226, 2375, 3261, 2824, 2974, 3005, 2951, 2357, 2581, 3098 ], "unit_k_proj_g14": [ 1731 ], "unit_o_proj_g05": [ 1749, 1446, 1417, 1280, 1369, 1752, 1594 ], "unit_o_proj_g30": [ 1300, 1678, 1805, 1567, 1327, 2159 ], "unit_q_proj_g20": [ 1647 ], "unit_k_proj_g06": [ 1780 ], "unit_v_proj_g24": [ 1944, 1883, 2956 ], "unit_v_proj_g16": [ 1677, 3448, 2007, 2090, 3329, 2055, 2949, 1916, 2485, 2403 ], "unit_q_proj_g18": [ 1777 ], "unit_q_proj_g08": [ 1734 ], "unit_gate_proj_g28": [ 2389, 3158, 2308, 2527 ], "unit_q_proj_g16": [ 2061, 1961 ], "unit_up_proj_g06": [ 2227, 2685, 2190, 3035 ], "unit_down_proj_g09": [ 1814 ], "unit_o_proj_g02": [ 1276, 2197, 1676, 1548, 1287, 1467 ], "unit_down_proj_g16": [ 1867, 2210 ], "unit_k_proj_g28": [ 2675 ], "unit_down_proj_g20": [ 1845, 1961, 2458 ] }, "dropped": { "unit_o_proj_g09": [ 1210, 1211, 1212, 1213, 1214, 1215, 1216, 1217, 1218, 1219 ], "unit_v_proj_g30": [ 1629, 1630, 1631, 1632, 1633, 1634 ], "unit_o_proj_g14": [ 1214, 1215, 1216, 1217 ], "unit_down_proj_g18": [ 1780, 1781, 1782, 1783, 1784, 1785 ], "unit_o_proj_g19": [ 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227 ], "unit_down_proj_g17": [ 1770, 1771, 1772, 1773, 1774, 1775, 1776, 1777 ], "unit_o_proj_g20": [ 1223, 1224 ], "unit_o_proj_g23": [ 1223, 1224, 1225, 1226, 1227 ], "unit_o_proj_g08": [ 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211, 1212, 1213, 1214, 1215, 1216, 1217, 1218 ], "unit_o_proj_g18": [ 1211, 1212, 1213, 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1221 ], "unit_o_proj_g21": [ 1216, 1217, 1218, 1219, 1220, 1221, 1222 ], "unit_o_proj_g27": [ 1225, 1226, 1227 ], "unit_v_proj_g26": [ 1618, 1619, 1620, 1621, 1622, 1623, 1624, 1625, 1626, 1627, 1628, 1629, 1630, 1631 ], "unit_down_proj_g10": [ 1787, 1788 ], "unit_down_proj_g12": [ 1772, 1773, 1774, 1775, 1776, 1777, 1778, 1779, 1780 ], "unit_v_proj_g06": [ 1620, 1621, 1622, 1623 ], "unit_gate_proj_g14": [ 2061, 2062, 2063, 2064, 2065, 2066, 2067, 2068, 2069 ], "unit_gate_proj_g04": [ 2069, 2070, 2071 ], "unit_v_proj_g02": [ 1624, 1625, 1626, 1627, 1628, 1629, 1630, 1631, 1632, 1633, 1634, 1635 ], "unit_gate_proj_g20": [ 2066, 2067, 2068, 2069, 2070 ], "unit_up_proj_g14": [ 2058, 2059, 2060, 2061, 2062, 2063 ], "unit_o_proj_g17": [ 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223, 1224 ], "unit_down_proj_g28": [ 1788, 1789 ], "unit_down_proj_g04": [ 1770, 1771, 1772, 1773, 1774, 1775, 1776, 1777, 1778, 1779, 1780, 1781, 1782, 1783, 1784, 1785 ], "unit_o_proj_g22": [ 1225, 1226 ], "unit_v_proj_g28": [ 1630, 1631, 1632, 1633 ], "unit_v_proj_g18": [ 1630, 1631, 1632, 1633, 1634 ], "unit_up_proj_g10": [ 2060, 2061, 2062, 2063 ], "unit_down_proj_g25": [ 1786, 1787, 1788 ], "unit_v_proj_g12": [ 1621, 1622 ], "unit_o_proj_g06": [ 1214, 1215, 1216 ], "unit_down_proj_g26": [ 1787, 1788, 1789 ], "unit_down_proj_g11": [ 1782, 1783, 1784, 1785, 1786, 1787 ], "unit_gate_proj_g12": [ 2068, 2069 ], "unit_o_proj_g31": [ 1224, 1225, 1226, 1227 ], "unit_gate_proj_g02": [ 2070, 2071 ], "unit_gate_proj_g30": [ 2066, 2067, 2068, 2069 ], "unit_gate_proj_g08": [ 2065, 2066, 2067, 2068, 2069, 2070 ], "unit_o_proj_g03": [ 1218, 1219, 1220 ], "unit_down_proj_g05": [ 1774, 1775, 1776, 1777, 1778, 1779, 1780, 1781, 1782 ], "unit_o_proj_g29": [ 1226 ], "unit_v_proj_g10": [ 1612, 1613, 1614, 1615, 1616, 1617, 1618, 1619, 1620, 1621, 1622, 1623, 1624, 1625, 1626, 1627, 1628, 1629 ], "unit_o_proj_g16": [ 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223, 1224, 1225 ], "unit_down_proj_g22": [ 1781, 1782, 1783, 1784, 1785, 1786, 1787, 1788 ], "unit_up_proj_g26": [ 2068, 2069, 2070, 2071 ], "unit_gate_proj_g26": [ 2071 ], "unit_o_proj_g28": [ 1208, 1209, 1210, 1211, 1212, 1213, 1214, 1215, 1216, 1217, 1218 ], "unit_o_proj_g15": [ 1215, 1216, 1217, 1218, 1219 ], "unit_o_proj_g07": [ 1199, 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211, 1212, 1213, 1214 ], "unit_o_proj_g25": [ 1223, 1224, 1225 ], "unit_down_proj_g30": [ 1788, 1789 ], "unit_gate_proj_g10": [ 2067, 2068 ], "unit_down_proj_g00": [ 1789 ], "unit_up_proj_g18": [ 2061, 2062, 2063, 2064, 2065, 2066, 2067, 2068, 2069 ], "unit_down_proj_g24": [ 1784, 1785 ], "unit_down_proj_g27": [ 1789 ], "unit_k_proj_g30": [ 1635, 1636, 1637 ], "unit_up_proj_g22": [ 2064, 2065, 2066, 2067, 2068 ], "unit_down_proj_g29": [ 1783, 1784, 1785 ], "unit_up_proj_g02": [ 2069 ], "unit_up_proj_g20": [ 2066 ], "unit_down_proj_g23": [ 1789, 1790 ], "unit_up_proj_g30": [ 2067, 2068, 2069 ], "unit_o_proj_g11": [ 1215, 1216, 1217, 1218, 1219, 1220, 1221 ], "unit_v_proj_g22": [ 1632, 1633, 1634 ], "unit_q_proj_g14": [ 1637 ], "unit_k_proj_g18": [ 1636, 1637 ], "unit_gate_proj_g16": [ 2062, 2063, 2064 ], "unit_down_proj_g15": [ 1771, 1772, 1773, 1774, 1775, 1776, 1777, 1778, 1779, 1780, 1781, 1782, 1783, 1784, 1785, 1786 ], "unit_q_proj_g30": [ 1637 ], "unit_v_proj_g04": [ 1608, 1609, 1610, 1611, 1612, 1613, 1614, 1615, 1616, 1617, 1618, 1619, 1620, 1621, 1622, 1623, 1624, 1625, 1626, 1627, 1628, 1629, 1630, 1631, 1632, 1633 ], "unit_k_proj_g14": [ 1637 ], "unit_o_proj_g05": [ 1221, 1222, 1223, 1224, 1225, 1226, 1227 ], "unit_o_proj_g30": [ 1220, 1221, 1222, 1223, 1224, 1225 ], "unit_q_proj_g20": [ 1637 ], "unit_k_proj_g06": [ 1637 ], "unit_v_proj_g24": [ 1633, 1634, 1635 ], "unit_v_proj_g16": [ 1617, 1618, 1619, 1620, 1621, 1622, 1623, 1624, 1625, 1626 ], "unit_q_proj_g18": [ 1637 ], "unit_q_proj_g08": [ 1637 ], "unit_gate_proj_g28": [ 2065, 2066, 2067, 2068 ], "unit_q_proj_g16": [ 1633, 1634 ], "unit_up_proj_g06": [ 2062, 2063, 2064, 2065 ], "unit_down_proj_g09": [ 1772 ], "unit_o_proj_g02": [ 1221, 1222, 1223, 1224, 1225, 1226 ], "unit_down_proj_g16": [ 1782, 1783 ], "unit_k_proj_g28": [ 1637 ], "unit_down_proj_g20": [ 1779, 1780, 1781 ] }, "added_by_type": { "o_proj": 165, "v_proj": 107, "down_proj": 105, "gate_proj": 41, "up_proj": 37, "k_proj": 8, "q_proj": 7 }, "spent": 12948736, "param_fraction": 0.5999091746275906, "predicted_orr": -0.15819761732018378, "predicted_orr_total": -0.32461821016369186, "tau_binding": false, "measured": { "refusal_nll": 2.554681406021118, "answer_nll": 0.8039287114143372, "direct_loss": 3.3586101174354552, "dsafe": 1.277449963092804, "dsafe_sem": 0.06748057762246237, "orr": -2.692242422103882, "orr_sem": 0.05752465563135103, "refh": -2.554681406021118, "refh_sem": 0.03789699756736986, "refb": -3.496171133518219, "refb_sem": 0.037862480132145396, "gap": -0.941489727497101, "gap_sem": 0.05357004598073761 }, "predicted_insert_value": 0.4724117534351535, "predicted_removal_value": 0.027596300475124735, "insert_cancellation": { "sum_abs_layer_value": 0.475532925396692, "abs_sum_value": 0.472411755064968, "n_opposite_sign": 13 }, "evict_cancellation": { "sum_abs_layer_value": 0.14180451041465858, "abs_sum_value": 0.12185735189268598, "n_opposite_sign": 107 }, "seconds": 112.18983221054077 } ] }