program(1.0) [buildInfo = dict, tensor>({{"coremlc-component-MIL", "3520.4.1"}, {"coremlc-version", "3520.5.1"}})] { func main(tensor attention_mask, tensor input_ids) [FlexibleShapeInformation = tuple, dict, tensor>>, tuple, dict, list, ?>>>>((("DefaultShapes", {{"attention_mask", [1, 1]}, {"input_ids", [1, 1]}}), ("RangeDims", {{"attention_mask", [[1, 1], [1, 1024]]}, {"input_ids", [[1, 1], [1, 1024]]}})))] { tensor encoder_embed_tokens_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(64))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(262355072))), name = tensor("encoder_embed_tokens_weight_palettized"), shape = tensor([256206, 1024])]; tensor encoder_embed_positions_weights_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(262356160))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(263406848))), name = tensor("encoder_embed_positions_weights_palettized"), shape = tensor([1026, 1024])]; tensor encoder_layers_0_self_attn_layer_norm_bias = const()[name = tensor("encoder_layers_0_self_attn_layer_norm_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(263407936)))]; tensor encoder_layers_0_self_attn_layer_norm_weight = const()[name = tensor("encoder_layers_0_self_attn_layer_norm_weight"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(263412096)))]; tensor encoder_layers_0_self_attn_q_proj_bias = const()[name = tensor("encoder_layers_0_self_attn_q_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(263416256)))]; tensor encoder_layers_0_self_attn_q_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(263420416))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(264469056))), name = tensor("encoder_layers_0_self_attn_q_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_0_self_attn_k_proj_bias = const()[name = tensor("encoder_layers_0_self_attn_k_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(264470144)))]; tensor encoder_layers_0_self_attn_k_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(264474304))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(265522944))), name = tensor("encoder_layers_0_self_attn_k_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_0_self_attn_v_proj_bias = const()[name = tensor("encoder_layers_0_self_attn_v_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(265524032)))]; tensor encoder_layers_0_self_attn_v_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(265528192))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(266576832))), name = tensor("encoder_layers_0_self_attn_v_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_0_self_attn_out_proj_bias = const()[name = tensor("encoder_layers_0_self_attn_out_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(266577920)))]; tensor encoder_layers_0_self_attn_out_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(266582080))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(267630720))), name = tensor("encoder_layers_0_self_attn_out_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_0_final_layer_norm_bias = const()[name = tensor("encoder_layers_0_final_layer_norm_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(267631808)))]; tensor encoder_layers_0_final_layer_norm_weight = const()[name = tensor("encoder_layers_0_final_layer_norm_weight"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(267635968)))]; tensor encoder_layers_0_fc1_bias = const()[name = tensor("encoder_layers_0_fc1_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(267640128)))]; tensor encoder_layers_0_fc1_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(267656576))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(271850944))), name = tensor("encoder_layers_0_fc1_weight_palettized"), shape = tensor([4096, 1024])]; tensor encoder_layers_0_fc2_bias = const()[name = tensor("encoder_layers_0_fc2_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(271852032)))]; tensor encoder_layers_0_fc2_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(271856192))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(276050560))), name = tensor("encoder_layers_0_fc2_weight_palettized"), shape = tensor([1024, 4096])]; tensor encoder_layers_1_self_attn_layer_norm_bias = const()[name = tensor("encoder_layers_1_self_attn_layer_norm_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(276051648)))]; tensor encoder_layers_1_self_attn_layer_norm_weight = const()[name = tensor("encoder_layers_1_self_attn_layer_norm_weight"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(276055808)))]; tensor encoder_layers_1_self_attn_q_proj_bias = const()[name = tensor("encoder_layers_1_self_attn_q_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(276059968)))]; tensor encoder_layers_1_self_attn_q_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(276064128))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(277112768))), name = tensor("encoder_layers_1_self_attn_q_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_1_self_attn_k_proj_bias = const()[name = tensor("encoder_layers_1_self_attn_k_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(277113856)))]; tensor encoder_layers_1_self_attn_k_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(277118016))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(278166656))), name = tensor("encoder_layers_1_self_attn_k_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_1_self_attn_v_proj_bias = const()[name = tensor("encoder_layers_1_self_attn_v_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(278167744)))]; tensor encoder_layers_1_self_attn_v_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(278171904))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(279220544))), name = tensor("encoder_layers_1_self_attn_v_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_1_self_attn_out_proj_bias = const()[name = tensor("encoder_layers_1_self_attn_out_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(279221632)))]; tensor encoder_layers_1_self_attn_out_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(279225792))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(280274432))), name = tensor("encoder_layers_1_self_attn_out_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_1_final_layer_norm_bias = const()[name = tensor("encoder_layers_1_final_layer_norm_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(280275520)))]; tensor encoder_layers_1_final_layer_norm_weight = const()[name = tensor("encoder_layers_1_final_layer_norm_weight"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(280279680)))]; tensor encoder_layers_1_fc1_bias = const()[name = tensor("encoder_layers_1_fc1_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(280283840)))]; tensor encoder_layers_1_fc1_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(280300288))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(284494656))), name = tensor("encoder_layers_1_fc1_weight_palettized"), shape = tensor([4096, 1024])]; tensor encoder_layers_1_fc2_bias = const()[name = tensor("encoder_layers_1_fc2_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(284495744)))]; tensor encoder_layers_1_fc2_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(284499904))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(288694272))), name = tensor("encoder_layers_1_fc2_weight_palettized"), shape = tensor([1024, 4096])]; tensor encoder_layers_2_self_attn_layer_norm_bias = const()[name = tensor("encoder_layers_2_self_attn_layer_norm_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(288695360)))]; tensor encoder_layers_2_self_attn_layer_norm_weight = const()[name = tensor("encoder_layers_2_self_attn_layer_norm_weight"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(288699520)))]; tensor encoder_layers_2_self_attn_q_proj_bias = const()[name = tensor("encoder_layers_2_self_attn_q_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(288703680)))]; tensor encoder_layers_2_self_attn_q_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(288707840))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(289756480))), name = tensor("encoder_layers_2_self_attn_q_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_2_self_attn_k_proj_bias = const()[name = tensor("encoder_layers_2_self_attn_k_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(289757568)))]; tensor encoder_layers_2_self_attn_k_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(289761728))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(290810368))), name = tensor("encoder_layers_2_self_attn_k_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_2_self_attn_v_proj_bias = const()[name = tensor("encoder_layers_2_self_attn_v_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(290811456)))]; tensor encoder_layers_2_self_attn_v_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(290815616))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(291864256))), name = tensor("encoder_layers_2_self_attn_v_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_2_self_attn_out_proj_bias = const()[name = tensor("encoder_layers_2_self_attn_out_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(291865344)))]; tensor encoder_layers_2_self_attn_out_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(291869504))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(292918144))), name = tensor("encoder_layers_2_self_attn_out_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_2_final_layer_norm_bias = const()[name = tensor("encoder_layers_2_final_layer_norm_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(292919232)))]; tensor encoder_layers_2_final_layer_norm_weight = const()[name = tensor("encoder_layers_2_final_layer_norm_weight"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(292923392)))]; tensor encoder_layers_2_fc1_bias = const()[name = tensor("encoder_layers_2_fc1_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(292927552)))]; tensor encoder_layers_2_fc1_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(292944000))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(297138368))), name = tensor("encoder_layers_2_fc1_weight_palettized"), shape = tensor([4096, 1024])]; tensor encoder_layers_2_fc2_bias = const()[name = tensor("encoder_layers_2_fc2_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(297139456)))]; tensor encoder_layers_2_fc2_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(297143616))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(301337984))), name = tensor("encoder_layers_2_fc2_weight_palettized"), shape = tensor([1024, 4096])]; tensor encoder_layers_3_self_attn_layer_norm_bias = const()[name = tensor("encoder_layers_3_self_attn_layer_norm_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(301339072)))]; tensor encoder_layers_3_self_attn_layer_norm_weight = const()[name = tensor("encoder_layers_3_self_attn_layer_norm_weight"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(301343232)))]; tensor encoder_layers_3_self_attn_q_proj_bias = const()[name = tensor("encoder_layers_3_self_attn_q_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(301347392)))]; tensor encoder_layers_3_self_attn_q_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(301351552))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(302400192))), name = tensor("encoder_layers_3_self_attn_q_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_3_self_attn_k_proj_bias = const()[name = tensor("encoder_layers_3_self_attn_k_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(302401280)))]; tensor encoder_layers_3_self_attn_k_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(302405440))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(303454080))), name = tensor("encoder_layers_3_self_attn_k_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_3_self_attn_v_proj_bias = const()[name = tensor("encoder_layers_3_self_attn_v_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(303455168)))]; tensor encoder_layers_3_self_attn_v_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(303459328))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(304507968))), name = tensor("encoder_layers_3_self_attn_v_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_3_self_attn_out_proj_bias = const()[name = tensor("encoder_layers_3_self_attn_out_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(304509056)))]; tensor encoder_layers_3_self_attn_out_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(304513216))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(305561856))), name = tensor("encoder_layers_3_self_attn_out_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_3_final_layer_norm_bias = const()[name = tensor("encoder_layers_3_final_layer_norm_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(305562944)))]; tensor encoder_layers_3_final_layer_norm_weight = const()[name = tensor("encoder_layers_3_final_layer_norm_weight"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(305567104)))]; tensor encoder_layers_3_fc1_bias = const()[name = tensor("encoder_layers_3_fc1_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(305571264)))]; tensor encoder_layers_3_fc1_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(305587712))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(309782080))), name = tensor("encoder_layers_3_fc1_weight_palettized"), shape = tensor([4096, 1024])]; tensor encoder_layers_3_fc2_bias = const()[name = tensor("encoder_layers_3_fc2_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(309783168)))]; tensor encoder_layers_3_fc2_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(309787328))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(313981696))), name = tensor("encoder_layers_3_fc2_weight_palettized"), shape = tensor([1024, 4096])]; tensor encoder_layers_4_self_attn_layer_norm_bias = const()[name = tensor("encoder_layers_4_self_attn_layer_norm_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(313982784)))]; tensor encoder_layers_4_self_attn_layer_norm_weight = const()[name = tensor("encoder_layers_4_self_attn_layer_norm_weight"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(313986944)))]; tensor encoder_layers_4_self_attn_q_proj_bias = const()[name = tensor("encoder_layers_4_self_attn_q_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(313991104)))]; tensor encoder_layers_4_self_attn_q_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(313995264))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(263406848))), name = tensor("encoder_layers_4_self_attn_q_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_4_self_attn_k_proj_bias = const()[name = tensor("encoder_layers_4_self_attn_k_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(315043904)))]; tensor encoder_layers_4_self_attn_k_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(315048064))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(316096704))), name = tensor("encoder_layers_4_self_attn_k_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_4_self_attn_v_proj_bias = const()[name = tensor("encoder_layers_4_self_attn_v_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(316097792)))]; tensor encoder_layers_4_self_attn_v_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(316101952))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(317150592))), name = tensor("encoder_layers_4_self_attn_v_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_4_self_attn_out_proj_bias = const()[name = tensor("encoder_layers_4_self_attn_out_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(317151680)))]; tensor encoder_layers_4_self_attn_out_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(317155840))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(318204480))), name = tensor("encoder_layers_4_self_attn_out_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_4_final_layer_norm_bias = const()[name = tensor("encoder_layers_4_final_layer_norm_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(318205568)))]; tensor encoder_layers_4_final_layer_norm_weight = const()[name = tensor("encoder_layers_4_final_layer_norm_weight"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(318209728)))]; tensor encoder_layers_4_fc1_bias = const()[name = tensor("encoder_layers_4_fc1_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(318213888)))]; tensor encoder_layers_4_fc1_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(318230336))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(322424704))), name = tensor("encoder_layers_4_fc1_weight_palettized"), shape = tensor([4096, 1024])]; tensor encoder_layers_4_fc2_bias = const()[name = tensor("encoder_layers_4_fc2_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(322425792)))]; tensor encoder_layers_4_fc2_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(322429952))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(326624320))), name = tensor("encoder_layers_4_fc2_weight_palettized"), shape = tensor([1024, 4096])]; tensor encoder_layers_5_self_attn_layer_norm_bias = const()[name = tensor("encoder_layers_5_self_attn_layer_norm_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(326625408)))]; tensor encoder_layers_5_self_attn_layer_norm_weight = const()[name = tensor("encoder_layers_5_self_attn_layer_norm_weight"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(326629568)))]; tensor encoder_layers_5_self_attn_q_proj_bias = const()[name = tensor("encoder_layers_5_self_attn_q_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(326633728)))]; tensor encoder_layers_5_self_attn_q_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(326637888))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(327686528))), name = tensor("encoder_layers_5_self_attn_q_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_5_self_attn_k_proj_bias = const()[name = tensor("encoder_layers_5_self_attn_k_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(327687616)))]; tensor encoder_layers_5_self_attn_k_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(327691776))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(328740416))), name = tensor("encoder_layers_5_self_attn_k_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_5_self_attn_v_proj_bias = const()[name = tensor("encoder_layers_5_self_attn_v_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(328741504)))]; tensor encoder_layers_5_self_attn_v_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(328745664))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(329794304))), name = tensor("encoder_layers_5_self_attn_v_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_5_self_attn_out_proj_bias = const()[name = tensor("encoder_layers_5_self_attn_out_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(329795392)))]; tensor encoder_layers_5_self_attn_out_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(329799552))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(330848192))), name = tensor("encoder_layers_5_self_attn_out_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_5_final_layer_norm_bias = const()[name = tensor("encoder_layers_5_final_layer_norm_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(330849280)))]; tensor encoder_layers_5_final_layer_norm_weight = const()[name = tensor("encoder_layers_5_final_layer_norm_weight"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(330853440)))]; tensor encoder_layers_5_fc1_bias = const()[name = tensor("encoder_layers_5_fc1_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(330857600)))]; tensor encoder_layers_5_fc1_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(330874048))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(335068416))), name = tensor("encoder_layers_5_fc1_weight_palettized"), shape = tensor([4096, 1024])]; tensor encoder_layers_5_fc2_bias = const()[name = tensor("encoder_layers_5_fc2_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(335069504)))]; tensor encoder_layers_5_fc2_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(335073664))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(339268032))), name = tensor("encoder_layers_5_fc2_weight_palettized"), shape = tensor([1024, 4096])]; tensor encoder_layers_6_self_attn_layer_norm_bias = const()[name = tensor("encoder_layers_6_self_attn_layer_norm_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(339269120)))]; tensor encoder_layers_6_self_attn_layer_norm_weight = const()[name = tensor("encoder_layers_6_self_attn_layer_norm_weight"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(339273280)))]; tensor encoder_layers_6_self_attn_q_proj_bias = const()[name = tensor("encoder_layers_6_self_attn_q_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(339277440)))]; tensor encoder_layers_6_self_attn_q_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(339281600))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(340330240))), name = tensor("encoder_layers_6_self_attn_q_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_6_self_attn_k_proj_bias = const()[name = tensor("encoder_layers_6_self_attn_k_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(340331328)))]; tensor encoder_layers_6_self_attn_k_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(340335488))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(341384128))), name = tensor("encoder_layers_6_self_attn_k_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_6_self_attn_v_proj_bias = const()[name = tensor("encoder_layers_6_self_attn_v_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(341385216)))]; tensor encoder_layers_6_self_attn_v_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(341389376))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(342438016))), name = tensor("encoder_layers_6_self_attn_v_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_6_self_attn_out_proj_bias = const()[name = tensor("encoder_layers_6_self_attn_out_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(342439104)))]; tensor encoder_layers_6_self_attn_out_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(342443264))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(343491904))), name = tensor("encoder_layers_6_self_attn_out_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_6_final_layer_norm_bias = const()[name = tensor("encoder_layers_6_final_layer_norm_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(343492992)))]; tensor encoder_layers_6_final_layer_norm_weight = const()[name = tensor("encoder_layers_6_final_layer_norm_weight"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(343497152)))]; tensor encoder_layers_6_fc1_bias = const()[name = tensor("encoder_layers_6_fc1_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(343501312)))]; tensor encoder_layers_6_fc1_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(343517760))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(347712128))), name = tensor("encoder_layers_6_fc1_weight_palettized"), shape = tensor([4096, 1024])]; tensor encoder_layers_6_fc2_bias = const()[name = tensor("encoder_layers_6_fc2_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(347713216)))]; tensor encoder_layers_6_fc2_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(347717376))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(351911744))), name = tensor("encoder_layers_6_fc2_weight_palettized"), shape = tensor([1024, 4096])]; tensor encoder_layers_7_self_attn_layer_norm_bias = const()[name = tensor("encoder_layers_7_self_attn_layer_norm_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(351912832)))]; tensor encoder_layers_7_self_attn_layer_norm_weight = const()[name = tensor("encoder_layers_7_self_attn_layer_norm_weight"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(351916992)))]; tensor encoder_layers_7_self_attn_q_proj_bias = const()[name = tensor("encoder_layers_7_self_attn_q_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(351921152)))]; tensor encoder_layers_7_self_attn_q_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(351925312))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(352973952))), name = tensor("encoder_layers_7_self_attn_q_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_7_self_attn_k_proj_bias = const()[name = tensor("encoder_layers_7_self_attn_k_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(352975040)))]; tensor encoder_layers_7_self_attn_k_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(352979200))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(354027840))), name = tensor("encoder_layers_7_self_attn_k_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_7_self_attn_v_proj_bias = const()[name = tensor("encoder_layers_7_self_attn_v_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(354028928)))]; tensor encoder_layers_7_self_attn_v_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(354033088))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(263406848))), name = tensor("encoder_layers_7_self_attn_v_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_7_self_attn_out_proj_bias = const()[name = tensor("encoder_layers_7_self_attn_out_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(355081728)))]; tensor encoder_layers_7_self_attn_out_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(355085888))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(356134528))), name = tensor("encoder_layers_7_self_attn_out_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_7_final_layer_norm_bias = const()[name = tensor("encoder_layers_7_final_layer_norm_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(356135616)))]; tensor encoder_layers_7_final_layer_norm_weight = const()[name = tensor("encoder_layers_7_final_layer_norm_weight"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(356139776)))]; tensor encoder_layers_7_fc1_bias = const()[name = tensor("encoder_layers_7_fc1_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(356143936)))]; tensor encoder_layers_7_fc1_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(356160384))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(360354752))), name = tensor("encoder_layers_7_fc1_weight_palettized"), shape = tensor([4096, 1024])]; tensor encoder_layers_7_fc2_bias = const()[name = tensor("encoder_layers_7_fc2_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(360355840)))]; tensor encoder_layers_7_fc2_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(360360000))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(364554368))), name = tensor("encoder_layers_7_fc2_weight_palettized"), shape = tensor([1024, 4096])]; tensor encoder_layers_8_self_attn_layer_norm_bias = const()[name = tensor("encoder_layers_8_self_attn_layer_norm_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(364555456)))]; tensor encoder_layers_8_self_attn_layer_norm_weight = const()[name = tensor("encoder_layers_8_self_attn_layer_norm_weight"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(364559616)))]; tensor encoder_layers_8_self_attn_q_proj_bias = const()[name = tensor("encoder_layers_8_self_attn_q_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(364563776)))]; tensor encoder_layers_8_self_attn_q_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(364567936))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(365616576))), name = tensor("encoder_layers_8_self_attn_q_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_8_self_attn_k_proj_bias = const()[name = tensor("encoder_layers_8_self_attn_k_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(365617664)))]; tensor encoder_layers_8_self_attn_k_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(365621824))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(366670464))), name = tensor("encoder_layers_8_self_attn_k_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_8_self_attn_v_proj_bias = const()[name = tensor("encoder_layers_8_self_attn_v_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(366671552)))]; tensor encoder_layers_8_self_attn_v_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(366675712))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(367724352))), name = tensor("encoder_layers_8_self_attn_v_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_8_self_attn_out_proj_bias = const()[name = tensor("encoder_layers_8_self_attn_out_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(367725440)))]; tensor encoder_layers_8_self_attn_out_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(367729600))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(368778240))), name = tensor("encoder_layers_8_self_attn_out_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_8_final_layer_norm_bias = const()[name = tensor("encoder_layers_8_final_layer_norm_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(368779328)))]; tensor encoder_layers_8_final_layer_norm_weight = const()[name = tensor("encoder_layers_8_final_layer_norm_weight"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(368783488)))]; tensor encoder_layers_8_fc1_bias = const()[name = tensor("encoder_layers_8_fc1_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(368787648)))]; tensor encoder_layers_8_fc1_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(368804096))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(372998464))), name = tensor("encoder_layers_8_fc1_weight_palettized"), shape = tensor([4096, 1024])]; tensor encoder_layers_8_fc2_bias = const()[name = tensor("encoder_layers_8_fc2_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(372999552)))]; tensor encoder_layers_8_fc2_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(373003712))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(267630720))), name = tensor("encoder_layers_8_fc2_weight_palettized"), shape = tensor([1024, 4096])]; tensor encoder_layers_9_self_attn_layer_norm_bias = const()[name = tensor("encoder_layers_9_self_attn_layer_norm_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(377198080)))]; tensor encoder_layers_9_self_attn_layer_norm_weight = const()[name = tensor("encoder_layers_9_self_attn_layer_norm_weight"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(377202240)))]; tensor encoder_layers_9_self_attn_q_proj_bias = const()[name = tensor("encoder_layers_9_self_attn_q_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(377206400)))]; tensor encoder_layers_9_self_attn_q_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(377210560))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(378259200))), name = tensor("encoder_layers_9_self_attn_q_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_9_self_attn_k_proj_bias = const()[name = tensor("encoder_layers_9_self_attn_k_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(378260288)))]; tensor encoder_layers_9_self_attn_k_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(378264448))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(379313088))), name = tensor("encoder_layers_9_self_attn_k_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_9_self_attn_v_proj_bias = const()[name = tensor("encoder_layers_9_self_attn_v_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(379314176)))]; tensor encoder_layers_9_self_attn_v_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(379318336))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(380366976))), name = tensor("encoder_layers_9_self_attn_v_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_9_self_attn_out_proj_bias = const()[name = tensor("encoder_layers_9_self_attn_out_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(380368064)))]; tensor encoder_layers_9_self_attn_out_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(380372224))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(381420864))), name = tensor("encoder_layers_9_self_attn_out_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_9_final_layer_norm_bias = const()[name = tensor("encoder_layers_9_final_layer_norm_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(381421952)))]; tensor encoder_layers_9_final_layer_norm_weight = const()[name = tensor("encoder_layers_9_final_layer_norm_weight"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(381426112)))]; tensor encoder_layers_9_fc1_bias = const()[name = tensor("encoder_layers_9_fc1_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(381430272)))]; tensor encoder_layers_9_fc1_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(381446720))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(385641088))), name = tensor("encoder_layers_9_fc1_weight_palettized"), shape = tensor([4096, 1024])]; tensor encoder_layers_9_fc2_bias = const()[name = tensor("encoder_layers_9_fc2_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(385642176)))]; tensor encoder_layers_9_fc2_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(385646336))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(280274432))), name = tensor("encoder_layers_9_fc2_weight_palettized"), shape = tensor([1024, 4096])]; tensor encoder_layers_10_self_attn_layer_norm_bias = const()[name = tensor("encoder_layers_10_self_attn_layer_norm_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(389840704)))]; tensor encoder_layers_10_self_attn_layer_norm_weight = const()[name = tensor("encoder_layers_10_self_attn_layer_norm_weight"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(389844864)))]; tensor encoder_layers_10_self_attn_q_proj_bias = const()[name = tensor("encoder_layers_10_self_attn_q_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(389849024)))]; tensor encoder_layers_10_self_attn_q_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(389853184))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(390901824))), name = tensor("encoder_layers_10_self_attn_q_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_10_self_attn_k_proj_bias = const()[name = tensor("encoder_layers_10_self_attn_k_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(390902912)))]; tensor encoder_layers_10_self_attn_k_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(390907072))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(391955712))), name = tensor("encoder_layers_10_self_attn_k_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_10_self_attn_v_proj_bias = const()[name = tensor("encoder_layers_10_self_attn_v_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(391956800)))]; tensor encoder_layers_10_self_attn_v_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(391960960))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(393009600))), name = tensor("encoder_layers_10_self_attn_v_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_10_self_attn_out_proj_bias = const()[name = tensor("encoder_layers_10_self_attn_out_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(393010688)))]; tensor encoder_layers_10_self_attn_out_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(393014848))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(394063488))), name = tensor("encoder_layers_10_self_attn_out_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_10_final_layer_norm_bias = const()[name = tensor("encoder_layers_10_final_layer_norm_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(394064576)))]; tensor encoder_layers_10_final_layer_norm_weight = const()[name = tensor("encoder_layers_10_final_layer_norm_weight"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(394068736)))]; tensor encoder_layers_10_fc1_bias = const()[name = tensor("encoder_layers_10_fc1_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(394072896)))]; tensor encoder_layers_10_fc1_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(394089344))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(398283712))), name = tensor("encoder_layers_10_fc1_weight_palettized"), shape = tensor([4096, 1024])]; tensor encoder_layers_10_fc2_bias = const()[name = tensor("encoder_layers_10_fc2_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(398284800)))]; tensor encoder_layers_10_fc2_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(398288960))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(402483328))), name = tensor("encoder_layers_10_fc2_weight_palettized"), shape = tensor([1024, 4096])]; tensor encoder_layers_11_self_attn_layer_norm_bias = const()[name = tensor("encoder_layers_11_self_attn_layer_norm_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(402484416)))]; tensor encoder_layers_11_self_attn_layer_norm_weight = const()[name = tensor("encoder_layers_11_self_attn_layer_norm_weight"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(402488576)))]; tensor encoder_layers_11_self_attn_q_proj_bias = const()[name = tensor("encoder_layers_11_self_attn_q_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(402492736)))]; tensor encoder_layers_11_self_attn_q_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(402496896))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(403545536))), name = tensor("encoder_layers_11_self_attn_q_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_11_self_attn_k_proj_bias = const()[name = tensor("encoder_layers_11_self_attn_k_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(403546624)))]; tensor encoder_layers_11_self_attn_k_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(403550784))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(404599424))), name = tensor("encoder_layers_11_self_attn_k_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_11_self_attn_v_proj_bias = const()[name = tensor("encoder_layers_11_self_attn_v_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(404600512)))]; tensor encoder_layers_11_self_attn_v_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(404604672))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(405653312))), name = tensor("encoder_layers_11_self_attn_v_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_11_self_attn_out_proj_bias = const()[name = tensor("encoder_layers_11_self_attn_out_proj_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(405654400)))]; tensor encoder_layers_11_self_attn_out_proj_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(405658560))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(406707200))), name = tensor("encoder_layers_11_self_attn_out_proj_weight_palettized"), shape = tensor([1024, 1024])]; tensor encoder_layers_11_final_layer_norm_bias = const()[name = tensor("encoder_layers_11_final_layer_norm_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(406708288)))]; tensor encoder_layers_11_final_layer_norm_weight = const()[name = tensor("encoder_layers_11_final_layer_norm_weight"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(406712448)))]; tensor encoder_layers_11_fc1_bias = const()[name = tensor("encoder_layers_11_fc1_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(406716608)))]; tensor encoder_layers_11_fc1_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(406733056))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(410927424))), name = tensor("encoder_layers_11_fc1_weight_palettized"), shape = tensor([4096, 1024])]; tensor encoder_layers_11_fc2_bias = const()[name = tensor("encoder_layers_11_fc2_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(410928512)))]; tensor encoder_layers_11_fc2_weight_palettized = constexpr_lut_to_dense()[indices = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(410932672))), lut = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(415127040))), name = tensor("encoder_layers_11_fc2_weight_palettized"), shape = tensor([1024, 4096])]; tensor encoder_layer_norm_bias = const()[name = tensor("encoder_layer_norm_bias"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(415128128)))]; tensor encoder_layer_norm_weight = const()[name = tensor("encoder_layer_norm_weight"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(415132288)))]; tensor var_5 = const()[name = tensor("op_5"), val = tensor(0x1.4f8b58p-17)]; tensor var_7 = const()[name = tensor("op_7"), val = tensor(0x1p-3)]; tensor var_10 = const()[name = tensor("op_10"), val = tensor(-0x1.fffffep+127)]; tensor var_17 = const()[name = tensor("op_17"), val = tensor(0)]; tensor var_23 = const()[name = tensor("op_23"), val = tensor(-1)]; tensor var_24 = const()[name = tensor("op_24"), val = tensor(1)]; tensor var_56_axis_0 = const()[name = tensor("op_56_axis_0"), val = tensor(0)]; tensor var_56_batch_dims_0 = const()[name = tensor("op_56_batch_dims_0"), val = tensor(0)]; tensor var_56 = gather(axis = var_56_axis_0, batch_dims = var_56_batch_dims_0, indices = input_ids, x = encoder_embed_tokens_weight_palettized)[name = tensor("op_56")]; tensor var_57 = const()[name = tensor("op_57"), val = tensor(0x1p+5)]; tensor inputs_embeds = mul(x = var_56, y = var_57)[name = tensor("inputs_embeds")]; tensor var_62 = not_equal(x = input_ids, y = var_24)[name = tensor("op_62")]; tensor mask_dtype_0 = const()[name = tensor("mask_dtype_0"), val = tensor("int32")]; tensor var_64_exclusive_0 = const()[name = tensor("op_64_exclusive_0"), val = tensor(false)]; tensor var_64_reverse_0 = const()[name = tensor("op_64_reverse_0"), val = tensor(false)]; tensor mask = cast(dtype = mask_dtype_0, x = var_62)[name = tensor("cast_2")]; tensor var_64 = cumsum(axis = var_24, exclusive = var_64_exclusive_0, reverse = var_64_reverse_0, x = mask)[name = tensor("op_64")]; tensor incremental_indices = mul(x = var_64, y = mask)[name = tensor("incremental_indices")]; tensor var_70 = const()[name = tensor("op_70"), val = tensor(1)]; tensor var_71 = add(x = incremental_indices, y = var_70)[name = tensor("op_71")]; tensor var_73 = const()[name = tensor("op_73"), val = tensor([-1])]; tensor var_74 = reshape(shape = var_73, x = var_71)[name = tensor("op_74")]; tensor var_75_batch_dims_0 = const()[name = tensor("op_75_batch_dims_0"), val = tensor(0)]; tensor var_75 = gather(axis = var_17, batch_dims = var_75_batch_dims_0, indices = var_74, x = encoder_embed_positions_weights_palettized)[name = tensor("op_75")]; tensor concat_1x = const()[name = tensor("concat_1x"), val = tensor([1, -1, 1024])]; tensor var_78 = reshape(shape = concat_1x, x = var_75)[name = tensor("op_78")]; tensor input_3 = add(x = inputs_embeds, y = var_78)[name = tensor("input_3")]; tensor var_83_shape = shape(x = attention_mask)[name = tensor("op_83_shape")]; tensor gather_3_indices_0 = const()[name = tensor("gather_3_indices_0"), val = tensor(1)]; tensor gather_3_axis_0 = const()[name = tensor("gather_3_axis_0"), val = tensor(0)]; tensor gather_3_batch_dims_0 = const()[name = tensor("gather_3_batch_dims_0"), val = tensor(0)]; tensor gather_3 = gather(axis = gather_3_axis_0, batch_dims = gather_3_batch_dims_0, indices = gather_3_indices_0, x = var_83_shape)[name = tensor("gather_3")]; tensor gather_4 = const()[name = tensor("gather_4"), val = tensor(1)]; tensor var_87_axes_0 = const()[name = tensor("op_87_axes_0"), val = tensor([1])]; tensor var_87 = expand_dims(axes = var_87_axes_0, x = attention_mask)[name = tensor("op_87")]; tensor var_88_axes_0 = const()[name = tensor("op_88_axes_0"), val = tensor([2])]; tensor var_88 = expand_dims(axes = var_88_axes_0, x = var_87)[name = tensor("op_88")]; tensor concat_2_axis_0 = const()[name = tensor("concat_2_axis_0"), val = tensor(0)]; tensor concat_2_interleave_0 = const()[name = tensor("concat_2_interleave_0"), val = tensor(false)]; tensor concat_2 = concat(axis = concat_2_axis_0, interleave = concat_2_interleave_0, values = (gather_4, var_24, gather_3, gather_3))[name = tensor("concat_2")]; tensor shape_0 = shape(x = var_88)[name = tensor("shape_0")]; tensor equal_0_y_0 = const()[name = tensor("equal_0_y_0"), val = tensor(-1)]; tensor equal_0 = equal(x = concat_2, y = equal_0_y_0)[name = tensor("equal_0")]; tensor select_0 = select(a = shape_0, b = concat_2, cond = equal_0)[name = tensor("select_0")]; tensor real_div_0 = real_div(x = select_0, y = shape_0)[name = tensor("real_div_0")]; tensor var_91 = tile(reps = real_div_0, x = var_88)[name = tensor("op_91")]; tensor expanded_mask_dtype_0 = const()[name = tensor("expanded_mask_dtype_0"), val = tensor("fp32")]; tensor const_1 = const()[name = tensor("const_1"), val = tensor(0x1p+0)]; tensor expanded_mask = cast(dtype = expanded_mask_dtype_0, x = var_91)[name = tensor("cast_1")]; tensor inverted_mask = sub(x = const_1, y = expanded_mask)[name = tensor("inverted_mask")]; tensor var_96_dtype_0 = const()[name = tensor("op_96_dtype_0"), val = tensor("bool")]; tensor var_96 = cast(dtype = var_96_dtype_0, x = inverted_mask)[name = tensor("cast_0")]; tensor attention_mask_3 = select(a = var_10, b = inverted_mask, cond = var_96)[name = tensor("attention_mask_3")]; tensor hidden_states_1_axes_0 = const()[name = tensor("hidden_states_1_axes_0"), val = tensor([-1])]; tensor hidden_states_1 = layer_norm(axes = hidden_states_1_axes_0, beta = encoder_layers_0_self_attn_layer_norm_bias, epsilon = var_5, gamma = encoder_layers_0_self_attn_layer_norm_weight, x = input_3)[name = tensor("hidden_states_1")]; tensor var_111_shape = shape(x = hidden_states_1)[name = tensor("op_111_shape")]; tensor gather_6 = const()[name = tensor("gather_6"), val = tensor(1)]; tensor gather_7_indices_0 = const()[name = tensor("gather_7_indices_0"), val = tensor(1)]; tensor gather_7_axis_0 = const()[name = tensor("gather_7_axis_0"), val = tensor(0)]; tensor gather_7_batch_dims_0 = const()[name = tensor("gather_7_batch_dims_0"), val = tensor(0)]; tensor gather_7 = gather(axis = gather_7_axis_0, batch_dims = gather_7_batch_dims_0, indices = gather_7_indices_0, x = var_111_shape)[name = tensor("gather_7")]; tensor var_115 = linear(bias = encoder_layers_0_self_attn_q_proj_bias, weight = encoder_layers_0_self_attn_q_proj_weight_palettized, x = hidden_states_1)[name = tensor("linear_0")]; tensor concat_3x = const()[name = tensor("concat_3x"), val = tensor([1, -1, 16, 64])]; tensor var_117 = reshape(shape = concat_3x, x = var_115)[name = tensor("op_117")]; tensor query_1_perm_0 = const()[name = tensor("query_1_perm_0"), val = tensor([0, 2, 1, 3])]; tensor key_states_1 = linear(bias = encoder_layers_0_self_attn_k_proj_bias, weight = encoder_layers_0_self_attn_k_proj_weight_palettized, x = hidden_states_1)[name = tensor("linear_1")]; tensor value_states_1 = linear(bias = encoder_layers_0_self_attn_v_proj_bias, weight = encoder_layers_0_self_attn_v_proj_weight_palettized, x = hidden_states_1)[name = tensor("linear_2")]; tensor concat_4x = const()[name = tensor("concat_4x"), val = tensor([1, -1, 16, 64])]; tensor var_126 = reshape(shape = concat_4x, x = key_states_1)[name = tensor("op_126")]; tensor key_1_perm_0 = const()[name = tensor("key_1_perm_0"), val = tensor([0, 2, -3, -1])]; tensor concat_5x = const()[name = tensor("concat_5x"), val = tensor([1, -1, 16, 64])]; tensor var_129 = reshape(shape = concat_5x, x = value_states_1)[name = tensor("op_129")]; tensor value_1_perm_0 = const()[name = tensor("value_1_perm_0"), val = tensor([0, 2, -3, -1])]; tensor key_1 = transpose(perm = key_1_perm_0, x = var_126)[name = tensor("transpose_47")]; tensor var_131_shape = shape(x = key_1)[name = tensor("op_131_shape")]; tensor gather_8_indices_0 = const()[name = tensor("gather_8_indices_0"), val = tensor(2)]; tensor gather_8_axis_0 = const()[name = tensor("gather_8_axis_0"), val = tensor(0)]; tensor gather_8_batch_dims_0 = const()[name = tensor("gather_8_batch_dims_0"), val = tensor(0)]; tensor gather_8 = gather(axis = gather_8_axis_0, batch_dims = gather_8_batch_dims_0, indices = gather_8_indices_0, x = var_131_shape)[name = tensor("gather_8")]; tensor concat_6_values0_0 = const()[name = tensor("concat_6_values0_0"), val = tensor(0)]; tensor concat_6_values1_0 = const()[name = tensor("concat_6_values1_0"), val = tensor(0)]; tensor concat_6_values2_0 = const()[name = tensor("concat_6_values2_0"), val = tensor(0)]; tensor concat_6_axis_0 = const()[name = tensor("concat_6_axis_0"), val = tensor(0)]; tensor concat_6_interleave_0 = const()[name = tensor("concat_6_interleave_0"), val = tensor(false)]; tensor concat_6 = concat(axis = concat_6_axis_0, interleave = concat_6_interleave_0, values = (concat_6_values0_0, concat_6_values1_0, concat_6_values2_0, gather_8))[name = tensor("concat_6")]; tensor attention_mask_5_begin_0 = const()[name = tensor("attention_mask_5_begin_0"), val = tensor([0, 0, 0, 0])]; tensor attention_mask_5_end_mask_0 = const()[name = tensor("attention_mask_5_end_mask_0"), val = tensor([true, true, true, false])]; tensor attention_mask_5 = slice_by_index(begin = attention_mask_5_begin_0, end = concat_6, end_mask = attention_mask_5_end_mask_0, x = attention_mask_3)[name = tensor("attention_mask_5")]; tensor query_1 = transpose(perm = query_1_perm_0, x = var_117)[name = tensor("transpose_46")]; tensor mul_0 = mul(x = query_1, y = var_7)[name = tensor("mul_0")]; tensor matmul_0_transpose_y_0 = const()[name = tensor("matmul_0_transpose_y_0"), val = tensor(true)]; tensor matmul_0_transpose_x_0 = const()[name = tensor("matmul_0_transpose_x_0"), val = tensor(false)]; tensor matmul_0 = matmul(transpose_x = matmul_0_transpose_x_0, transpose_y = matmul_0_transpose_y_0, x = mul_0, y = key_1)[name = tensor("matmul_0")]; tensor add_0 = add(x = matmul_0, y = attention_mask_5)[name = tensor("add_0")]; tensor softmax_0_axis_0 = const()[name = tensor("softmax_0_axis_0"), val = tensor(-1)]; tensor softmax_0 = softmax(axis = softmax_0_axis_0, x = add_0)[name = tensor("softmax_0")]; tensor attn_output_1_transpose_x_0 = const()[name = tensor("attn_output_1_transpose_x_0"), val = tensor(false)]; tensor attn_output_1_transpose_y_0 = const()[name = tensor("attn_output_1_transpose_y_0"), val = tensor(false)]; tensor value_1 = transpose(perm = value_1_perm_0, x = var_129)[name = tensor("transpose_45")]; tensor attn_output_1 = matmul(transpose_x = attn_output_1_transpose_x_0, transpose_y = attn_output_1_transpose_y_0, x = softmax_0, y = value_1)[name = tensor("attn_output_1")]; tensor var_137_perm_0 = const()[name = tensor("op_137_perm_0"), val = tensor([0, 2, 1, 3])]; tensor concat_7_axis_0 = const()[name = tensor("concat_7_axis_0"), val = tensor(0)]; tensor concat_7_interleave_0 = const()[name = tensor("concat_7_interleave_0"), val = tensor(false)]; tensor concat_7 = concat(axis = concat_7_axis_0, interleave = concat_7_interleave_0, values = (gather_6, gather_7, var_23))[name = tensor("concat_7")]; tensor var_137 = transpose(perm = var_137_perm_0, x = attn_output_1)[name = tensor("transpose_44")]; tensor var_140 = reshape(shape = concat_7, x = var_137)[name = tensor("op_140")]; tensor input_9 = linear(bias = encoder_layers_0_self_attn_out_proj_bias, weight = encoder_layers_0_self_attn_out_proj_weight_palettized, x = var_140)[name = tensor("linear_3")]; tensor input_11 = add(x = input_3, y = input_9)[name = tensor("input_11")]; tensor input_13_axes_0 = const()[name = tensor("input_13_axes_0"), val = tensor([-1])]; tensor input_13 = layer_norm(axes = input_13_axes_0, beta = encoder_layers_0_final_layer_norm_bias, epsilon = var_5, gamma = encoder_layers_0_final_layer_norm_weight, x = input_11)[name = tensor("input_13")]; tensor input_15 = linear(bias = encoder_layers_0_fc1_bias, weight = encoder_layers_0_fc1_weight_palettized, x = input_13)[name = tensor("linear_4")]; tensor input_17 = relu(x = input_15)[name = tensor("input_17")]; tensor input_21 = linear(bias = encoder_layers_0_fc2_bias, weight = encoder_layers_0_fc2_weight_palettized, x = input_17)[name = tensor("linear_5")]; tensor input_23 = add(x = input_11, y = input_21)[name = tensor("input_23")]; tensor hidden_states_7_axes_0 = const()[name = tensor("hidden_states_7_axes_0"), val = tensor([-1])]; tensor hidden_states_7 = layer_norm(axes = hidden_states_7_axes_0, beta = encoder_layers_1_self_attn_layer_norm_bias, epsilon = var_5, gamma = encoder_layers_1_self_attn_layer_norm_weight, x = input_23)[name = tensor("hidden_states_7")]; tensor var_174_shape = shape(x = hidden_states_7)[name = tensor("op_174_shape")]; tensor gather_9 = const()[name = tensor("gather_9"), val = tensor(1)]; tensor gather_10_indices_0 = const()[name = tensor("gather_10_indices_0"), val = tensor(1)]; tensor gather_10_axis_0 = const()[name = tensor("gather_10_axis_0"), val = tensor(0)]; tensor gather_10_batch_dims_0 = const()[name = tensor("gather_10_batch_dims_0"), val = tensor(0)]; tensor gather_10 = gather(axis = gather_10_axis_0, batch_dims = gather_10_batch_dims_0, indices = gather_10_indices_0, x = var_174_shape)[name = tensor("gather_10")]; tensor var_178 = linear(bias = encoder_layers_1_self_attn_q_proj_bias, weight = encoder_layers_1_self_attn_q_proj_weight_palettized, x = hidden_states_7)[name = tensor("linear_6")]; tensor concat_8x = const()[name = tensor("concat_8x"), val = tensor([1, -1, 16, 64])]; tensor var_180 = reshape(shape = concat_8x, x = var_178)[name = tensor("op_180")]; tensor query_3_perm_0 = const()[name = tensor("query_3_perm_0"), val = tensor([0, 2, 1, 3])]; tensor key_states_3 = linear(bias = encoder_layers_1_self_attn_k_proj_bias, weight = encoder_layers_1_self_attn_k_proj_weight_palettized, x = hidden_states_7)[name = tensor("linear_7")]; tensor value_states_3 = linear(bias = encoder_layers_1_self_attn_v_proj_bias, weight = encoder_layers_1_self_attn_v_proj_weight_palettized, x = hidden_states_7)[name = tensor("linear_8")]; tensor concat_9x = const()[name = tensor("concat_9x"), val = tensor([1, -1, 16, 64])]; tensor var_189 = reshape(shape = concat_9x, x = key_states_3)[name = tensor("op_189")]; tensor key_3_perm_0 = const()[name = tensor("key_3_perm_0"), val = tensor([0, 2, -3, -1])]; tensor concat_10x = const()[name = tensor("concat_10x"), val = tensor([1, -1, 16, 64])]; tensor var_192 = reshape(shape = concat_10x, x = value_states_3)[name = tensor("op_192")]; tensor value_3_perm_0 = const()[name = tensor("value_3_perm_0"), val = tensor([0, 2, -3, -1])]; tensor key_3 = transpose(perm = key_3_perm_0, x = var_189)[name = tensor("transpose_43")]; tensor var_194_shape = shape(x = key_3)[name = tensor("op_194_shape")]; tensor gather_11_indices_0 = const()[name = tensor("gather_11_indices_0"), val = tensor(2)]; tensor gather_11_axis_0 = const()[name = tensor("gather_11_axis_0"), val = tensor(0)]; tensor gather_11_batch_dims_0 = const()[name = tensor("gather_11_batch_dims_0"), val = tensor(0)]; tensor gather_11 = gather(axis = gather_11_axis_0, batch_dims = gather_11_batch_dims_0, indices = gather_11_indices_0, x = var_194_shape)[name = tensor("gather_11")]; tensor concat_11_values0_0 = const()[name = tensor("concat_11_values0_0"), val = tensor(0)]; tensor concat_11_values1_0 = const()[name = tensor("concat_11_values1_0"), val = tensor(0)]; tensor concat_11_values2_0 = const()[name = tensor("concat_11_values2_0"), val = tensor(0)]; tensor concat_11_axis_0 = const()[name = tensor("concat_11_axis_0"), val = tensor(0)]; tensor concat_11_interleave_0 = const()[name = tensor("concat_11_interleave_0"), val = tensor(false)]; tensor concat_11 = concat(axis = concat_11_axis_0, interleave = concat_11_interleave_0, values = (concat_11_values0_0, concat_11_values1_0, concat_11_values2_0, gather_11))[name = tensor("concat_11")]; tensor attention_mask_7_begin_0 = const()[name = tensor("attention_mask_7_begin_0"), val = tensor([0, 0, 0, 0])]; tensor attention_mask_7_end_mask_0 = const()[name = tensor("attention_mask_7_end_mask_0"), val = tensor([true, true, true, false])]; tensor attention_mask_7 = slice_by_index(begin = attention_mask_7_begin_0, end = concat_11, end_mask = attention_mask_7_end_mask_0, x = attention_mask_3)[name = tensor("attention_mask_7")]; tensor query_3 = transpose(perm = query_3_perm_0, x = var_180)[name = tensor("transpose_42")]; tensor mul_1 = mul(x = query_3, y = var_7)[name = tensor("mul_1")]; tensor matmul_1_transpose_y_0 = const()[name = tensor("matmul_1_transpose_y_0"), val = tensor(true)]; tensor matmul_1_transpose_x_0 = const()[name = tensor("matmul_1_transpose_x_0"), val = tensor(false)]; tensor matmul_1 = matmul(transpose_x = matmul_1_transpose_x_0, transpose_y = matmul_1_transpose_y_0, x = mul_1, y = key_3)[name = tensor("matmul_1")]; tensor add_1 = add(x = matmul_1, y = attention_mask_7)[name = tensor("add_1")]; tensor softmax_1_axis_0 = const()[name = tensor("softmax_1_axis_0"), val = tensor(-1)]; tensor softmax_1 = softmax(axis = softmax_1_axis_0, x = add_1)[name = tensor("softmax_1")]; tensor attn_output_5_transpose_x_0 = const()[name = tensor("attn_output_5_transpose_x_0"), val = tensor(false)]; tensor attn_output_5_transpose_y_0 = const()[name = tensor("attn_output_5_transpose_y_0"), val = tensor(false)]; tensor value_3 = transpose(perm = value_3_perm_0, x = var_192)[name = tensor("transpose_41")]; tensor attn_output_5 = matmul(transpose_x = attn_output_5_transpose_x_0, transpose_y = attn_output_5_transpose_y_0, x = softmax_1, y = value_3)[name = tensor("attn_output_5")]; tensor var_200_perm_0 = const()[name = tensor("op_200_perm_0"), val = tensor([0, 2, 1, 3])]; tensor concat_12_axis_0 = const()[name = tensor("concat_12_axis_0"), val = tensor(0)]; tensor concat_12_interleave_0 = const()[name = tensor("concat_12_interleave_0"), val = tensor(false)]; tensor concat_12 = concat(axis = concat_12_axis_0, interleave = concat_12_interleave_0, values = (gather_9, gather_10, var_23))[name = tensor("concat_12")]; tensor var_200 = transpose(perm = var_200_perm_0, x = attn_output_5)[name = tensor("transpose_40")]; tensor var_203 = reshape(shape = concat_12, x = var_200)[name = tensor("op_203")]; tensor input_27 = linear(bias = encoder_layers_1_self_attn_out_proj_bias, weight = encoder_layers_1_self_attn_out_proj_weight_palettized, x = var_203)[name = tensor("linear_9")]; tensor input_29 = add(x = input_23, y = input_27)[name = tensor("input_29")]; tensor input_31_axes_0 = const()[name = tensor("input_31_axes_0"), val = tensor([-1])]; tensor input_31 = layer_norm(axes = input_31_axes_0, beta = encoder_layers_1_final_layer_norm_bias, epsilon = var_5, gamma = encoder_layers_1_final_layer_norm_weight, x = input_29)[name = tensor("input_31")]; tensor input_33 = linear(bias = encoder_layers_1_fc1_bias, weight = encoder_layers_1_fc1_weight_palettized, x = input_31)[name = tensor("linear_10")]; tensor input_35 = relu(x = input_33)[name = tensor("input_35")]; tensor input_39 = linear(bias = encoder_layers_1_fc2_bias, weight = encoder_layers_1_fc2_weight_palettized, x = input_35)[name = tensor("linear_11")]; tensor input_41 = add(x = input_29, y = input_39)[name = tensor("input_41")]; tensor hidden_states_13_axes_0 = const()[name = tensor("hidden_states_13_axes_0"), val = tensor([-1])]; tensor hidden_states_13 = layer_norm(axes = hidden_states_13_axes_0, beta = encoder_layers_2_self_attn_layer_norm_bias, epsilon = var_5, gamma = encoder_layers_2_self_attn_layer_norm_weight, x = input_41)[name = tensor("hidden_states_13")]; tensor var_237_shape = shape(x = hidden_states_13)[name = tensor("op_237_shape")]; tensor gather_12 = const()[name = tensor("gather_12"), val = tensor(1)]; tensor gather_13_indices_0 = const()[name = tensor("gather_13_indices_0"), val = tensor(1)]; tensor gather_13_axis_0 = const()[name = tensor("gather_13_axis_0"), val = tensor(0)]; tensor gather_13_batch_dims_0 = const()[name = tensor("gather_13_batch_dims_0"), val = tensor(0)]; tensor gather_13 = gather(axis = gather_13_axis_0, batch_dims = gather_13_batch_dims_0, indices = gather_13_indices_0, x = var_237_shape)[name = tensor("gather_13")]; tensor var_241 = linear(bias = encoder_layers_2_self_attn_q_proj_bias, weight = encoder_layers_2_self_attn_q_proj_weight_palettized, x = hidden_states_13)[name = tensor("linear_12")]; tensor concat_13x = const()[name = tensor("concat_13x"), val = tensor([1, -1, 16, 64])]; tensor var_243 = reshape(shape = concat_13x, x = var_241)[name = tensor("op_243")]; tensor query_5_perm_0 = const()[name = tensor("query_5_perm_0"), val = tensor([0, 2, 1, 3])]; tensor key_states_5 = linear(bias = encoder_layers_2_self_attn_k_proj_bias, weight = encoder_layers_2_self_attn_k_proj_weight_palettized, x = hidden_states_13)[name = tensor("linear_13")]; tensor value_states_5 = linear(bias = encoder_layers_2_self_attn_v_proj_bias, weight = encoder_layers_2_self_attn_v_proj_weight_palettized, x = hidden_states_13)[name = tensor("linear_14")]; tensor concat_14x = const()[name = tensor("concat_14x"), val = tensor([1, -1, 16, 64])]; tensor var_252 = reshape(shape = concat_14x, x = key_states_5)[name = tensor("op_252")]; tensor key_5_perm_0 = const()[name = tensor("key_5_perm_0"), val = tensor([0, 2, -3, -1])]; tensor concat_15x = const()[name = tensor("concat_15x"), val = tensor([1, -1, 16, 64])]; tensor var_255 = reshape(shape = concat_15x, x = value_states_5)[name = tensor("op_255")]; tensor value_5_perm_0 = const()[name = tensor("value_5_perm_0"), val = tensor([0, 2, -3, -1])]; tensor key_5 = transpose(perm = key_5_perm_0, x = var_252)[name = tensor("transpose_39")]; tensor var_257_shape = shape(x = key_5)[name = tensor("op_257_shape")]; tensor gather_14_indices_0 = const()[name = tensor("gather_14_indices_0"), val = tensor(2)]; tensor gather_14_axis_0 = const()[name = tensor("gather_14_axis_0"), val = tensor(0)]; tensor gather_14_batch_dims_0 = const()[name = tensor("gather_14_batch_dims_0"), val = tensor(0)]; tensor gather_14 = gather(axis = gather_14_axis_0, batch_dims = gather_14_batch_dims_0, indices = gather_14_indices_0, x = var_257_shape)[name = tensor("gather_14")]; tensor concat_16_values0_0 = const()[name = tensor("concat_16_values0_0"), val = tensor(0)]; tensor concat_16_values1_0 = const()[name = tensor("concat_16_values1_0"), val = tensor(0)]; tensor concat_16_values2_0 = const()[name = tensor("concat_16_values2_0"), val = tensor(0)]; tensor concat_16_axis_0 = const()[name = tensor("concat_16_axis_0"), val = tensor(0)]; tensor concat_16_interleave_0 = const()[name = tensor("concat_16_interleave_0"), val = tensor(false)]; tensor concat_16 = concat(axis = concat_16_axis_0, interleave = concat_16_interleave_0, values = (concat_16_values0_0, concat_16_values1_0, concat_16_values2_0, gather_14))[name = tensor("concat_16")]; tensor attention_mask_9_begin_0 = const()[name = tensor("attention_mask_9_begin_0"), val = tensor([0, 0, 0, 0])]; tensor attention_mask_9_end_mask_0 = const()[name = tensor("attention_mask_9_end_mask_0"), val = tensor([true, true, true, false])]; tensor attention_mask_9 = slice_by_index(begin = attention_mask_9_begin_0, end = concat_16, end_mask = attention_mask_9_end_mask_0, x = attention_mask_3)[name = tensor("attention_mask_9")]; tensor query_5 = transpose(perm = query_5_perm_0, x = var_243)[name = tensor("transpose_38")]; tensor mul_2 = mul(x = query_5, y = var_7)[name = tensor("mul_2")]; tensor matmul_2_transpose_y_0 = const()[name = tensor("matmul_2_transpose_y_0"), val = tensor(true)]; tensor matmul_2_transpose_x_0 = const()[name = tensor("matmul_2_transpose_x_0"), val = tensor(false)]; tensor matmul_2 = matmul(transpose_x = matmul_2_transpose_x_0, transpose_y = matmul_2_transpose_y_0, x = mul_2, y = key_5)[name = tensor("matmul_2")]; tensor add_2 = add(x = matmul_2, y = attention_mask_9)[name = tensor("add_2")]; tensor softmax_2_axis_0 = const()[name = tensor("softmax_2_axis_0"), val = tensor(-1)]; tensor softmax_2 = softmax(axis = softmax_2_axis_0, x = add_2)[name = tensor("softmax_2")]; tensor attn_output_9_transpose_x_0 = const()[name = tensor("attn_output_9_transpose_x_0"), val = tensor(false)]; tensor attn_output_9_transpose_y_0 = const()[name = tensor("attn_output_9_transpose_y_0"), val = tensor(false)]; tensor value_5 = transpose(perm = value_5_perm_0, x = var_255)[name = tensor("transpose_37")]; tensor attn_output_9 = matmul(transpose_x = attn_output_9_transpose_x_0, transpose_y = attn_output_9_transpose_y_0, x = softmax_2, y = value_5)[name = tensor("attn_output_9")]; tensor var_263_perm_0 = const()[name = tensor("op_263_perm_0"), val = tensor([0, 2, 1, 3])]; tensor concat_17_axis_0 = const()[name = tensor("concat_17_axis_0"), val = tensor(0)]; tensor concat_17_interleave_0 = const()[name = tensor("concat_17_interleave_0"), val = tensor(false)]; tensor concat_17 = concat(axis = concat_17_axis_0, interleave = concat_17_interleave_0, values = (gather_12, gather_13, var_23))[name = tensor("concat_17")]; tensor var_263 = transpose(perm = var_263_perm_0, x = attn_output_9)[name = tensor("transpose_36")]; tensor var_266 = reshape(shape = concat_17, x = var_263)[name = tensor("op_266")]; tensor input_45 = linear(bias = encoder_layers_2_self_attn_out_proj_bias, weight = encoder_layers_2_self_attn_out_proj_weight_palettized, x = var_266)[name = tensor("linear_15")]; tensor input_47 = add(x = input_41, y = input_45)[name = tensor("input_47")]; tensor input_49_axes_0 = const()[name = tensor("input_49_axes_0"), val = tensor([-1])]; tensor input_49 = layer_norm(axes = input_49_axes_0, beta = encoder_layers_2_final_layer_norm_bias, epsilon = var_5, gamma = encoder_layers_2_final_layer_norm_weight, x = input_47)[name = tensor("input_49")]; tensor input_51 = linear(bias = encoder_layers_2_fc1_bias, weight = encoder_layers_2_fc1_weight_palettized, x = input_49)[name = tensor("linear_16")]; tensor input_53 = relu(x = input_51)[name = tensor("input_53")]; tensor input_57 = linear(bias = encoder_layers_2_fc2_bias, weight = encoder_layers_2_fc2_weight_palettized, x = input_53)[name = tensor("linear_17")]; tensor input_59 = add(x = input_47, y = input_57)[name = tensor("input_59")]; tensor hidden_states_19_axes_0 = const()[name = tensor("hidden_states_19_axes_0"), val = tensor([-1])]; tensor hidden_states_19 = layer_norm(axes = hidden_states_19_axes_0, beta = encoder_layers_3_self_attn_layer_norm_bias, epsilon = var_5, gamma = encoder_layers_3_self_attn_layer_norm_weight, x = input_59)[name = tensor("hidden_states_19")]; tensor var_300_shape = shape(x = hidden_states_19)[name = tensor("op_300_shape")]; tensor gather_15 = const()[name = tensor("gather_15"), val = tensor(1)]; tensor gather_16_indices_0 = const()[name = tensor("gather_16_indices_0"), val = tensor(1)]; tensor gather_16_axis_0 = const()[name = tensor("gather_16_axis_0"), val = tensor(0)]; tensor gather_16_batch_dims_0 = const()[name = tensor("gather_16_batch_dims_0"), val = tensor(0)]; tensor gather_16 = gather(axis = gather_16_axis_0, batch_dims = gather_16_batch_dims_0, indices = gather_16_indices_0, x = var_300_shape)[name = tensor("gather_16")]; tensor var_304 = linear(bias = encoder_layers_3_self_attn_q_proj_bias, weight = encoder_layers_3_self_attn_q_proj_weight_palettized, x = hidden_states_19)[name = tensor("linear_18")]; tensor concat_18x = const()[name = tensor("concat_18x"), val = tensor([1, -1, 16, 64])]; tensor var_306 = reshape(shape = concat_18x, x = var_304)[name = tensor("op_306")]; tensor query_7_perm_0 = const()[name = tensor("query_7_perm_0"), val = tensor([0, 2, 1, 3])]; tensor key_states_7 = linear(bias = encoder_layers_3_self_attn_k_proj_bias, weight = encoder_layers_3_self_attn_k_proj_weight_palettized, x = hidden_states_19)[name = tensor("linear_19")]; tensor value_states_7 = linear(bias = encoder_layers_3_self_attn_v_proj_bias, weight = encoder_layers_3_self_attn_v_proj_weight_palettized, x = hidden_states_19)[name = tensor("linear_20")]; tensor concat_19x = const()[name = tensor("concat_19x"), val = tensor([1, -1, 16, 64])]; tensor var_315 = reshape(shape = concat_19x, x = key_states_7)[name = tensor("op_315")]; tensor key_7_perm_0 = const()[name = tensor("key_7_perm_0"), val = tensor([0, 2, -3, -1])]; tensor concat_20x = const()[name = tensor("concat_20x"), val = tensor([1, -1, 16, 64])]; tensor var_318 = reshape(shape = concat_20x, x = value_states_7)[name = tensor("op_318")]; tensor value_7_perm_0 = const()[name = tensor("value_7_perm_0"), val = tensor([0, 2, -3, -1])]; tensor key_7 = transpose(perm = key_7_perm_0, x = var_315)[name = tensor("transpose_35")]; tensor var_320_shape = shape(x = key_7)[name = tensor("op_320_shape")]; tensor gather_17_indices_0 = const()[name = tensor("gather_17_indices_0"), val = tensor(2)]; tensor gather_17_axis_0 = const()[name = tensor("gather_17_axis_0"), val = tensor(0)]; tensor gather_17_batch_dims_0 = const()[name = tensor("gather_17_batch_dims_0"), val = tensor(0)]; tensor gather_17 = gather(axis = gather_17_axis_0, batch_dims = gather_17_batch_dims_0, indices = gather_17_indices_0, x = var_320_shape)[name = tensor("gather_17")]; tensor concat_21_values0_0 = const()[name = tensor("concat_21_values0_0"), val = tensor(0)]; tensor concat_21_values1_0 = const()[name = tensor("concat_21_values1_0"), val = tensor(0)]; tensor concat_21_values2_0 = const()[name = tensor("concat_21_values2_0"), val = tensor(0)]; tensor concat_21_axis_0 = const()[name = tensor("concat_21_axis_0"), val = tensor(0)]; tensor concat_21_interleave_0 = const()[name = tensor("concat_21_interleave_0"), val = tensor(false)]; tensor concat_21 = concat(axis = concat_21_axis_0, interleave = concat_21_interleave_0, values = (concat_21_values0_0, concat_21_values1_0, concat_21_values2_0, gather_17))[name = tensor("concat_21")]; tensor attention_mask_11_begin_0 = const()[name = tensor("attention_mask_11_begin_0"), val = tensor([0, 0, 0, 0])]; tensor attention_mask_11_end_mask_0 = const()[name = tensor("attention_mask_11_end_mask_0"), val = tensor([true, true, true, false])]; tensor attention_mask_11 = slice_by_index(begin = attention_mask_11_begin_0, end = concat_21, end_mask = attention_mask_11_end_mask_0, x = attention_mask_3)[name = tensor("attention_mask_11")]; tensor query_7 = transpose(perm = query_7_perm_0, x = var_306)[name = tensor("transpose_34")]; tensor mul_3 = mul(x = query_7, y = var_7)[name = tensor("mul_3")]; tensor matmul_3_transpose_y_0 = const()[name = tensor("matmul_3_transpose_y_0"), val = tensor(true)]; tensor matmul_3_transpose_x_0 = const()[name = tensor("matmul_3_transpose_x_0"), val = tensor(false)]; tensor matmul_3 = matmul(transpose_x = matmul_3_transpose_x_0, transpose_y = matmul_3_transpose_y_0, x = mul_3, y = key_7)[name = tensor("matmul_3")]; tensor add_3 = add(x = matmul_3, y = attention_mask_11)[name = tensor("add_3")]; tensor softmax_3_axis_0 = const()[name = tensor("softmax_3_axis_0"), val = tensor(-1)]; tensor softmax_3 = softmax(axis = softmax_3_axis_0, x = add_3)[name = tensor("softmax_3")]; tensor attn_output_13_transpose_x_0 = const()[name = tensor("attn_output_13_transpose_x_0"), val = tensor(false)]; tensor attn_output_13_transpose_y_0 = const()[name = tensor("attn_output_13_transpose_y_0"), val = tensor(false)]; tensor value_7 = transpose(perm = value_7_perm_0, x = var_318)[name = tensor("transpose_33")]; tensor attn_output_13 = matmul(transpose_x = attn_output_13_transpose_x_0, transpose_y = attn_output_13_transpose_y_0, x = softmax_3, y = value_7)[name = tensor("attn_output_13")]; tensor var_326_perm_0 = const()[name = tensor("op_326_perm_0"), val = tensor([0, 2, 1, 3])]; tensor concat_22_axis_0 = const()[name = tensor("concat_22_axis_0"), val = tensor(0)]; tensor concat_22_interleave_0 = const()[name = tensor("concat_22_interleave_0"), val = tensor(false)]; tensor concat_22 = concat(axis = concat_22_axis_0, interleave = concat_22_interleave_0, values = (gather_15, gather_16, var_23))[name = tensor("concat_22")]; tensor var_326 = transpose(perm = var_326_perm_0, x = attn_output_13)[name = tensor("transpose_32")]; tensor var_329 = reshape(shape = concat_22, x = var_326)[name = tensor("op_329")]; tensor input_63 = linear(bias = encoder_layers_3_self_attn_out_proj_bias, weight = encoder_layers_3_self_attn_out_proj_weight_palettized, x = var_329)[name = tensor("linear_21")]; tensor input_65 = add(x = input_59, y = input_63)[name = tensor("input_65")]; tensor input_67_axes_0 = const()[name = tensor("input_67_axes_0"), val = tensor([-1])]; tensor input_67 = layer_norm(axes = input_67_axes_0, beta = encoder_layers_3_final_layer_norm_bias, epsilon = var_5, gamma = encoder_layers_3_final_layer_norm_weight, x = input_65)[name = tensor("input_67")]; tensor input_69 = linear(bias = encoder_layers_3_fc1_bias, weight = encoder_layers_3_fc1_weight_palettized, x = input_67)[name = tensor("linear_22")]; tensor input_71 = relu(x = input_69)[name = tensor("input_71")]; tensor input_75 = linear(bias = encoder_layers_3_fc2_bias, weight = encoder_layers_3_fc2_weight_palettized, x = input_71)[name = tensor("linear_23")]; tensor input_77 = add(x = input_65, y = input_75)[name = tensor("input_77")]; tensor hidden_states_25_axes_0 = const()[name = tensor("hidden_states_25_axes_0"), val = tensor([-1])]; tensor hidden_states_25 = layer_norm(axes = hidden_states_25_axes_0, beta = encoder_layers_4_self_attn_layer_norm_bias, epsilon = var_5, gamma = encoder_layers_4_self_attn_layer_norm_weight, x = input_77)[name = tensor("hidden_states_25")]; tensor var_363_shape = shape(x = hidden_states_25)[name = tensor("op_363_shape")]; tensor gather_18 = const()[name = tensor("gather_18"), val = tensor(1)]; tensor gather_19_indices_0 = const()[name = tensor("gather_19_indices_0"), val = tensor(1)]; tensor gather_19_axis_0 = const()[name = tensor("gather_19_axis_0"), val = tensor(0)]; tensor gather_19_batch_dims_0 = const()[name = tensor("gather_19_batch_dims_0"), val = tensor(0)]; tensor gather_19 = gather(axis = gather_19_axis_0, batch_dims = gather_19_batch_dims_0, indices = gather_19_indices_0, x = var_363_shape)[name = tensor("gather_19")]; tensor var_367 = linear(bias = encoder_layers_4_self_attn_q_proj_bias, weight = encoder_layers_4_self_attn_q_proj_weight_palettized, x = hidden_states_25)[name = tensor("linear_24")]; tensor concat_23x = const()[name = tensor("concat_23x"), val = tensor([1, -1, 16, 64])]; tensor var_369 = reshape(shape = concat_23x, x = var_367)[name = tensor("op_369")]; tensor query_9_perm_0 = const()[name = tensor("query_9_perm_0"), val = tensor([0, 2, 1, 3])]; tensor key_states_9 = linear(bias = encoder_layers_4_self_attn_k_proj_bias, weight = encoder_layers_4_self_attn_k_proj_weight_palettized, x = hidden_states_25)[name = tensor("linear_25")]; tensor value_states_9 = linear(bias = encoder_layers_4_self_attn_v_proj_bias, weight = encoder_layers_4_self_attn_v_proj_weight_palettized, x = hidden_states_25)[name = tensor("linear_26")]; tensor concat_24x = const()[name = tensor("concat_24x"), val = tensor([1, -1, 16, 64])]; tensor var_378 = reshape(shape = concat_24x, x = key_states_9)[name = tensor("op_378")]; tensor key_9_perm_0 = const()[name = tensor("key_9_perm_0"), val = tensor([0, 2, -3, -1])]; tensor concat_25x = const()[name = tensor("concat_25x"), val = tensor([1, -1, 16, 64])]; tensor var_381 = reshape(shape = concat_25x, x = value_states_9)[name = tensor("op_381")]; tensor value_9_perm_0 = const()[name = tensor("value_9_perm_0"), val = tensor([0, 2, -3, -1])]; tensor key_9 = transpose(perm = key_9_perm_0, x = var_378)[name = tensor("transpose_31")]; tensor var_383_shape = shape(x = key_9)[name = tensor("op_383_shape")]; tensor gather_20_indices_0 = const()[name = tensor("gather_20_indices_0"), val = tensor(2)]; tensor gather_20_axis_0 = const()[name = tensor("gather_20_axis_0"), val = tensor(0)]; tensor gather_20_batch_dims_0 = const()[name = tensor("gather_20_batch_dims_0"), val = tensor(0)]; tensor gather_20 = gather(axis = gather_20_axis_0, batch_dims = gather_20_batch_dims_0, indices = gather_20_indices_0, x = var_383_shape)[name = tensor("gather_20")]; tensor concat_26_values0_0 = const()[name = tensor("concat_26_values0_0"), val = tensor(0)]; tensor concat_26_values1_0 = const()[name = tensor("concat_26_values1_0"), val = tensor(0)]; tensor concat_26_values2_0 = const()[name = tensor("concat_26_values2_0"), val = tensor(0)]; tensor concat_26_axis_0 = const()[name = tensor("concat_26_axis_0"), val = tensor(0)]; tensor concat_26_interleave_0 = const()[name = tensor("concat_26_interleave_0"), val = tensor(false)]; tensor concat_26 = concat(axis = concat_26_axis_0, interleave = concat_26_interleave_0, values = (concat_26_values0_0, concat_26_values1_0, concat_26_values2_0, gather_20))[name = tensor("concat_26")]; tensor attention_mask_13_begin_0 = const()[name = tensor("attention_mask_13_begin_0"), val = tensor([0, 0, 0, 0])]; tensor attention_mask_13_end_mask_0 = const()[name = tensor("attention_mask_13_end_mask_0"), val = tensor([true, true, true, false])]; tensor attention_mask_13 = slice_by_index(begin = attention_mask_13_begin_0, end = concat_26, end_mask = attention_mask_13_end_mask_0, x = attention_mask_3)[name = tensor("attention_mask_13")]; tensor query_9 = transpose(perm = query_9_perm_0, x = var_369)[name = tensor("transpose_30")]; tensor mul_4 = mul(x = query_9, y = var_7)[name = tensor("mul_4")]; tensor matmul_4_transpose_y_0 = const()[name = tensor("matmul_4_transpose_y_0"), val = tensor(true)]; tensor matmul_4_transpose_x_0 = const()[name = tensor("matmul_4_transpose_x_0"), val = tensor(false)]; tensor matmul_4 = matmul(transpose_x = matmul_4_transpose_x_0, transpose_y = matmul_4_transpose_y_0, x = mul_4, y = key_9)[name = tensor("matmul_4")]; tensor add_4 = add(x = matmul_4, y = attention_mask_13)[name = tensor("add_4")]; tensor softmax_4_axis_0 = const()[name = tensor("softmax_4_axis_0"), val = tensor(-1)]; tensor softmax_4 = softmax(axis = softmax_4_axis_0, x = add_4)[name = tensor("softmax_4")]; tensor attn_output_17_transpose_x_0 = const()[name = tensor("attn_output_17_transpose_x_0"), val = tensor(false)]; tensor attn_output_17_transpose_y_0 = const()[name = tensor("attn_output_17_transpose_y_0"), val = tensor(false)]; tensor value_9 = transpose(perm = value_9_perm_0, x = var_381)[name = tensor("transpose_29")]; tensor attn_output_17 = matmul(transpose_x = attn_output_17_transpose_x_0, transpose_y = attn_output_17_transpose_y_0, x = softmax_4, y = value_9)[name = tensor("attn_output_17")]; tensor var_389_perm_0 = const()[name = tensor("op_389_perm_0"), val = tensor([0, 2, 1, 3])]; tensor concat_27_axis_0 = const()[name = tensor("concat_27_axis_0"), val = tensor(0)]; tensor concat_27_interleave_0 = const()[name = tensor("concat_27_interleave_0"), val = tensor(false)]; tensor concat_27 = concat(axis = concat_27_axis_0, interleave = concat_27_interleave_0, values = (gather_18, gather_19, var_23))[name = tensor("concat_27")]; tensor var_389 = transpose(perm = var_389_perm_0, x = attn_output_17)[name = tensor("transpose_28")]; tensor var_392 = reshape(shape = concat_27, x = var_389)[name = tensor("op_392")]; tensor input_81 = linear(bias = encoder_layers_4_self_attn_out_proj_bias, weight = encoder_layers_4_self_attn_out_proj_weight_palettized, x = var_392)[name = tensor("linear_27")]; tensor input_83 = add(x = input_77, y = input_81)[name = tensor("input_83")]; tensor input_85_axes_0 = const()[name = tensor("input_85_axes_0"), val = tensor([-1])]; tensor input_85 = layer_norm(axes = input_85_axes_0, beta = encoder_layers_4_final_layer_norm_bias, epsilon = var_5, gamma = encoder_layers_4_final_layer_norm_weight, x = input_83)[name = tensor("input_85")]; tensor input_87 = linear(bias = encoder_layers_4_fc1_bias, weight = encoder_layers_4_fc1_weight_palettized, x = input_85)[name = tensor("linear_28")]; tensor input_89 = relu(x = input_87)[name = tensor("input_89")]; tensor input_93 = linear(bias = encoder_layers_4_fc2_bias, weight = encoder_layers_4_fc2_weight_palettized, x = input_89)[name = tensor("linear_29")]; tensor input_95 = add(x = input_83, y = input_93)[name = tensor("input_95")]; tensor hidden_states_31_axes_0 = const()[name = tensor("hidden_states_31_axes_0"), val = tensor([-1])]; tensor hidden_states_31 = layer_norm(axes = hidden_states_31_axes_0, beta = encoder_layers_5_self_attn_layer_norm_bias, epsilon = var_5, gamma = encoder_layers_5_self_attn_layer_norm_weight, x = input_95)[name = tensor("hidden_states_31")]; tensor var_426_shape = shape(x = hidden_states_31)[name = tensor("op_426_shape")]; tensor gather_21 = const()[name = tensor("gather_21"), val = tensor(1)]; tensor gather_22_indices_0 = const()[name = tensor("gather_22_indices_0"), val = tensor(1)]; tensor gather_22_axis_0 = const()[name = tensor("gather_22_axis_0"), val = tensor(0)]; tensor gather_22_batch_dims_0 = const()[name = tensor("gather_22_batch_dims_0"), val = tensor(0)]; tensor gather_22 = gather(axis = gather_22_axis_0, batch_dims = gather_22_batch_dims_0, indices = gather_22_indices_0, x = var_426_shape)[name = tensor("gather_22")]; tensor var_430 = linear(bias = encoder_layers_5_self_attn_q_proj_bias, weight = encoder_layers_5_self_attn_q_proj_weight_palettized, x = hidden_states_31)[name = tensor("linear_30")]; tensor concat_28x = const()[name = tensor("concat_28x"), val = tensor([1, -1, 16, 64])]; tensor var_432 = reshape(shape = concat_28x, x = var_430)[name = tensor("op_432")]; tensor query_11_perm_0 = const()[name = tensor("query_11_perm_0"), val = tensor([0, 2, 1, 3])]; tensor key_states_11 = linear(bias = encoder_layers_5_self_attn_k_proj_bias, weight = encoder_layers_5_self_attn_k_proj_weight_palettized, x = hidden_states_31)[name = tensor("linear_31")]; tensor value_states_11 = linear(bias = encoder_layers_5_self_attn_v_proj_bias, weight = encoder_layers_5_self_attn_v_proj_weight_palettized, x = hidden_states_31)[name = tensor("linear_32")]; tensor concat_29x = const()[name = tensor("concat_29x"), val = tensor([1, -1, 16, 64])]; tensor var_441 = reshape(shape = concat_29x, x = key_states_11)[name = tensor("op_441")]; tensor key_11_perm_0 = const()[name = tensor("key_11_perm_0"), val = tensor([0, 2, -3, -1])]; tensor concat_30x = const()[name = tensor("concat_30x"), val = tensor([1, -1, 16, 64])]; tensor var_444 = reshape(shape = concat_30x, x = value_states_11)[name = tensor("op_444")]; tensor value_11_perm_0 = const()[name = tensor("value_11_perm_0"), val = tensor([0, 2, -3, -1])]; tensor key_11 = transpose(perm = key_11_perm_0, x = var_441)[name = tensor("transpose_27")]; tensor var_446_shape = shape(x = key_11)[name = tensor("op_446_shape")]; tensor gather_23_indices_0 = const()[name = tensor("gather_23_indices_0"), val = tensor(2)]; tensor gather_23_axis_0 = const()[name = tensor("gather_23_axis_0"), val = tensor(0)]; tensor gather_23_batch_dims_0 = const()[name = tensor("gather_23_batch_dims_0"), val = tensor(0)]; tensor gather_23 = gather(axis = gather_23_axis_0, batch_dims = gather_23_batch_dims_0, indices = gather_23_indices_0, x = var_446_shape)[name = tensor("gather_23")]; tensor concat_31_values0_0 = const()[name = tensor("concat_31_values0_0"), val = tensor(0)]; tensor concat_31_values1_0 = const()[name = tensor("concat_31_values1_0"), val = tensor(0)]; tensor concat_31_values2_0 = const()[name = tensor("concat_31_values2_0"), val = tensor(0)]; tensor concat_31_axis_0 = const()[name = tensor("concat_31_axis_0"), val = tensor(0)]; tensor concat_31_interleave_0 = const()[name = tensor("concat_31_interleave_0"), val = tensor(false)]; tensor concat_31 = concat(axis = concat_31_axis_0, interleave = concat_31_interleave_0, values = (concat_31_values0_0, concat_31_values1_0, concat_31_values2_0, gather_23))[name = tensor("concat_31")]; tensor attention_mask_15_begin_0 = const()[name = tensor("attention_mask_15_begin_0"), val = tensor([0, 0, 0, 0])]; tensor attention_mask_15_end_mask_0 = const()[name = tensor("attention_mask_15_end_mask_0"), val = tensor([true, true, true, false])]; tensor attention_mask_15 = slice_by_index(begin = attention_mask_15_begin_0, end = concat_31, end_mask = attention_mask_15_end_mask_0, x = attention_mask_3)[name = tensor("attention_mask_15")]; tensor query_11 = transpose(perm = query_11_perm_0, x = var_432)[name = tensor("transpose_26")]; tensor mul_5 = mul(x = query_11, y = var_7)[name = tensor("mul_5")]; tensor matmul_5_transpose_y_0 = const()[name = tensor("matmul_5_transpose_y_0"), val = tensor(true)]; tensor matmul_5_transpose_x_0 = const()[name = tensor("matmul_5_transpose_x_0"), val = tensor(false)]; tensor matmul_5 = matmul(transpose_x = matmul_5_transpose_x_0, transpose_y = matmul_5_transpose_y_0, x = mul_5, y = key_11)[name = tensor("matmul_5")]; tensor add_5 = add(x = matmul_5, y = attention_mask_15)[name = tensor("add_5")]; tensor softmax_5_axis_0 = const()[name = tensor("softmax_5_axis_0"), val = tensor(-1)]; tensor softmax_5 = softmax(axis = softmax_5_axis_0, x = add_5)[name = tensor("softmax_5")]; tensor attn_output_21_transpose_x_0 = const()[name = tensor("attn_output_21_transpose_x_0"), val = tensor(false)]; tensor attn_output_21_transpose_y_0 = const()[name = tensor("attn_output_21_transpose_y_0"), val = tensor(false)]; tensor value_11 = transpose(perm = value_11_perm_0, x = var_444)[name = tensor("transpose_25")]; tensor attn_output_21 = matmul(transpose_x = attn_output_21_transpose_x_0, transpose_y = attn_output_21_transpose_y_0, x = softmax_5, y = value_11)[name = tensor("attn_output_21")]; tensor var_452_perm_0 = const()[name = tensor("op_452_perm_0"), val = tensor([0, 2, 1, 3])]; tensor concat_32_axis_0 = const()[name = tensor("concat_32_axis_0"), val = tensor(0)]; tensor concat_32_interleave_0 = const()[name = tensor("concat_32_interleave_0"), val = tensor(false)]; tensor concat_32 = concat(axis = concat_32_axis_0, interleave = concat_32_interleave_0, values = (gather_21, gather_22, var_23))[name = tensor("concat_32")]; tensor var_452 = transpose(perm = var_452_perm_0, x = attn_output_21)[name = tensor("transpose_24")]; tensor var_455 = reshape(shape = concat_32, x = var_452)[name = tensor("op_455")]; tensor input_99 = linear(bias = encoder_layers_5_self_attn_out_proj_bias, weight = encoder_layers_5_self_attn_out_proj_weight_palettized, x = var_455)[name = tensor("linear_33")]; tensor input_101 = add(x = input_95, y = input_99)[name = tensor("input_101")]; tensor input_103_axes_0 = const()[name = tensor("input_103_axes_0"), val = tensor([-1])]; tensor input_103 = layer_norm(axes = input_103_axes_0, beta = encoder_layers_5_final_layer_norm_bias, epsilon = var_5, gamma = encoder_layers_5_final_layer_norm_weight, x = input_101)[name = tensor("input_103")]; tensor input_105 = linear(bias = encoder_layers_5_fc1_bias, weight = encoder_layers_5_fc1_weight_palettized, x = input_103)[name = tensor("linear_34")]; tensor input_107 = relu(x = input_105)[name = tensor("input_107")]; tensor input_111 = linear(bias = encoder_layers_5_fc2_bias, weight = encoder_layers_5_fc2_weight_palettized, x = input_107)[name = tensor("linear_35")]; tensor input_113 = add(x = input_101, y = input_111)[name = tensor("input_113")]; tensor hidden_states_37_axes_0 = const()[name = tensor("hidden_states_37_axes_0"), val = tensor([-1])]; tensor hidden_states_37 = layer_norm(axes = hidden_states_37_axes_0, beta = encoder_layers_6_self_attn_layer_norm_bias, epsilon = var_5, gamma = encoder_layers_6_self_attn_layer_norm_weight, x = input_113)[name = tensor("hidden_states_37")]; tensor var_489_shape = shape(x = hidden_states_37)[name = tensor("op_489_shape")]; tensor gather_24 = const()[name = tensor("gather_24"), val = tensor(1)]; tensor gather_25_indices_0 = const()[name = tensor("gather_25_indices_0"), val = tensor(1)]; tensor gather_25_axis_0 = const()[name = tensor("gather_25_axis_0"), val = tensor(0)]; tensor gather_25_batch_dims_0 = const()[name = tensor("gather_25_batch_dims_0"), val = tensor(0)]; tensor gather_25 = gather(axis = gather_25_axis_0, batch_dims = gather_25_batch_dims_0, indices = gather_25_indices_0, x = var_489_shape)[name = tensor("gather_25")]; tensor var_493 = linear(bias = encoder_layers_6_self_attn_q_proj_bias, weight = encoder_layers_6_self_attn_q_proj_weight_palettized, x = hidden_states_37)[name = tensor("linear_36")]; tensor concat_33x = const()[name = tensor("concat_33x"), val = tensor([1, -1, 16, 64])]; tensor var_495 = reshape(shape = concat_33x, x = var_493)[name = tensor("op_495")]; tensor query_13_perm_0 = const()[name = tensor("query_13_perm_0"), val = tensor([0, 2, 1, 3])]; tensor key_states_13 = linear(bias = encoder_layers_6_self_attn_k_proj_bias, weight = encoder_layers_6_self_attn_k_proj_weight_palettized, x = hidden_states_37)[name = tensor("linear_37")]; tensor value_states_13 = linear(bias = encoder_layers_6_self_attn_v_proj_bias, weight = encoder_layers_6_self_attn_v_proj_weight_palettized, x = hidden_states_37)[name = tensor("linear_38")]; tensor concat_34x = const()[name = tensor("concat_34x"), val = tensor([1, -1, 16, 64])]; tensor var_504 = reshape(shape = concat_34x, x = key_states_13)[name = tensor("op_504")]; tensor key_13_perm_0 = const()[name = tensor("key_13_perm_0"), val = tensor([0, 2, -3, -1])]; tensor concat_35x = const()[name = tensor("concat_35x"), val = tensor([1, -1, 16, 64])]; tensor var_507 = reshape(shape = concat_35x, x = value_states_13)[name = tensor("op_507")]; tensor value_13_perm_0 = const()[name = tensor("value_13_perm_0"), val = tensor([0, 2, -3, -1])]; tensor key_13 = transpose(perm = key_13_perm_0, x = var_504)[name = tensor("transpose_23")]; tensor var_509_shape = shape(x = key_13)[name = tensor("op_509_shape")]; tensor gather_26_indices_0 = const()[name = tensor("gather_26_indices_0"), val = tensor(2)]; tensor gather_26_axis_0 = const()[name = tensor("gather_26_axis_0"), val = tensor(0)]; tensor gather_26_batch_dims_0 = const()[name = tensor("gather_26_batch_dims_0"), val = tensor(0)]; tensor gather_26 = gather(axis = gather_26_axis_0, batch_dims = gather_26_batch_dims_0, indices = gather_26_indices_0, x = var_509_shape)[name = tensor("gather_26")]; tensor concat_36_values0_0 = const()[name = tensor("concat_36_values0_0"), val = tensor(0)]; tensor concat_36_values1_0 = const()[name = tensor("concat_36_values1_0"), val = tensor(0)]; tensor concat_36_values2_0 = const()[name = tensor("concat_36_values2_0"), val = tensor(0)]; tensor concat_36_axis_0 = const()[name = tensor("concat_36_axis_0"), val = tensor(0)]; tensor concat_36_interleave_0 = const()[name = tensor("concat_36_interleave_0"), val = tensor(false)]; tensor concat_36 = concat(axis = concat_36_axis_0, interleave = concat_36_interleave_0, values = (concat_36_values0_0, concat_36_values1_0, concat_36_values2_0, gather_26))[name = tensor("concat_36")]; tensor attention_mask_17_begin_0 = const()[name = tensor("attention_mask_17_begin_0"), val = tensor([0, 0, 0, 0])]; tensor attention_mask_17_end_mask_0 = const()[name = tensor("attention_mask_17_end_mask_0"), val = tensor([true, true, true, false])]; tensor attention_mask_17 = slice_by_index(begin = attention_mask_17_begin_0, end = concat_36, end_mask = attention_mask_17_end_mask_0, x = attention_mask_3)[name = tensor("attention_mask_17")]; tensor query_13 = transpose(perm = query_13_perm_0, x = var_495)[name = tensor("transpose_22")]; tensor mul_6 = mul(x = query_13, y = var_7)[name = tensor("mul_6")]; tensor matmul_6_transpose_y_0 = const()[name = tensor("matmul_6_transpose_y_0"), val = tensor(true)]; tensor matmul_6_transpose_x_0 = const()[name = tensor("matmul_6_transpose_x_0"), val = tensor(false)]; tensor matmul_6 = matmul(transpose_x = matmul_6_transpose_x_0, transpose_y = matmul_6_transpose_y_0, x = mul_6, y = key_13)[name = tensor("matmul_6")]; tensor add_6 = add(x = matmul_6, y = attention_mask_17)[name = tensor("add_6")]; tensor softmax_6_axis_0 = const()[name = tensor("softmax_6_axis_0"), val = tensor(-1)]; tensor softmax_6 = softmax(axis = softmax_6_axis_0, x = add_6)[name = tensor("softmax_6")]; tensor attn_output_25_transpose_x_0 = const()[name = tensor("attn_output_25_transpose_x_0"), val = tensor(false)]; tensor attn_output_25_transpose_y_0 = const()[name = tensor("attn_output_25_transpose_y_0"), val = tensor(false)]; tensor value_13 = transpose(perm = value_13_perm_0, x = var_507)[name = tensor("transpose_21")]; tensor attn_output_25 = matmul(transpose_x = attn_output_25_transpose_x_0, transpose_y = attn_output_25_transpose_y_0, x = softmax_6, y = value_13)[name = tensor("attn_output_25")]; tensor var_515_perm_0 = const()[name = tensor("op_515_perm_0"), val = tensor([0, 2, 1, 3])]; tensor concat_37_axis_0 = const()[name = tensor("concat_37_axis_0"), val = tensor(0)]; tensor concat_37_interleave_0 = const()[name = tensor("concat_37_interleave_0"), val = tensor(false)]; tensor concat_37 = concat(axis = concat_37_axis_0, interleave = concat_37_interleave_0, values = (gather_24, gather_25, var_23))[name = tensor("concat_37")]; tensor var_515 = transpose(perm = var_515_perm_0, x = attn_output_25)[name = tensor("transpose_20")]; tensor var_518 = reshape(shape = concat_37, x = var_515)[name = tensor("op_518")]; tensor input_117 = linear(bias = encoder_layers_6_self_attn_out_proj_bias, weight = encoder_layers_6_self_attn_out_proj_weight_palettized, x = var_518)[name = tensor("linear_39")]; tensor input_119 = add(x = input_113, y = input_117)[name = tensor("input_119")]; tensor input_121_axes_0 = const()[name = tensor("input_121_axes_0"), val = tensor([-1])]; tensor input_121 = layer_norm(axes = input_121_axes_0, beta = encoder_layers_6_final_layer_norm_bias, epsilon = var_5, gamma = encoder_layers_6_final_layer_norm_weight, x = input_119)[name = tensor("input_121")]; tensor input_123 = linear(bias = encoder_layers_6_fc1_bias, weight = encoder_layers_6_fc1_weight_palettized, x = input_121)[name = tensor("linear_40")]; tensor input_125 = relu(x = input_123)[name = tensor("input_125")]; tensor input_129 = linear(bias = encoder_layers_6_fc2_bias, weight = encoder_layers_6_fc2_weight_palettized, x = input_125)[name = tensor("linear_41")]; tensor input_131 = add(x = input_119, y = input_129)[name = tensor("input_131")]; tensor hidden_states_43_axes_0 = const()[name = tensor("hidden_states_43_axes_0"), val = tensor([-1])]; tensor hidden_states_43 = layer_norm(axes = hidden_states_43_axes_0, beta = encoder_layers_7_self_attn_layer_norm_bias, epsilon = var_5, gamma = encoder_layers_7_self_attn_layer_norm_weight, x = input_131)[name = tensor("hidden_states_43")]; tensor var_552_shape = shape(x = hidden_states_43)[name = tensor("op_552_shape")]; tensor gather_27 = const()[name = tensor("gather_27"), val = tensor(1)]; tensor gather_28_indices_0 = const()[name = tensor("gather_28_indices_0"), val = tensor(1)]; tensor gather_28_axis_0 = const()[name = tensor("gather_28_axis_0"), val = tensor(0)]; tensor gather_28_batch_dims_0 = const()[name = tensor("gather_28_batch_dims_0"), val = tensor(0)]; tensor gather_28 = gather(axis = gather_28_axis_0, batch_dims = gather_28_batch_dims_0, indices = gather_28_indices_0, x = var_552_shape)[name = tensor("gather_28")]; tensor var_556 = linear(bias = encoder_layers_7_self_attn_q_proj_bias, weight = encoder_layers_7_self_attn_q_proj_weight_palettized, x = hidden_states_43)[name = tensor("linear_42")]; tensor concat_38x = const()[name = tensor("concat_38x"), val = tensor([1, -1, 16, 64])]; tensor var_558 = reshape(shape = concat_38x, x = var_556)[name = tensor("op_558")]; tensor query_15_perm_0 = const()[name = tensor("query_15_perm_0"), val = tensor([0, 2, 1, 3])]; tensor key_states_15 = linear(bias = encoder_layers_7_self_attn_k_proj_bias, weight = encoder_layers_7_self_attn_k_proj_weight_palettized, x = hidden_states_43)[name = tensor("linear_43")]; tensor value_states_15 = linear(bias = encoder_layers_7_self_attn_v_proj_bias, weight = encoder_layers_7_self_attn_v_proj_weight_palettized, x = hidden_states_43)[name = tensor("linear_44")]; tensor concat_39x = const()[name = tensor("concat_39x"), val = tensor([1, -1, 16, 64])]; tensor var_567 = reshape(shape = concat_39x, x = key_states_15)[name = tensor("op_567")]; tensor key_15_perm_0 = const()[name = tensor("key_15_perm_0"), val = tensor([0, 2, -3, -1])]; tensor concat_40x = const()[name = tensor("concat_40x"), val = tensor([1, -1, 16, 64])]; tensor var_570 = reshape(shape = concat_40x, x = value_states_15)[name = tensor("op_570")]; tensor value_15_perm_0 = const()[name = tensor("value_15_perm_0"), val = tensor([0, 2, -3, -1])]; tensor key_15 = transpose(perm = key_15_perm_0, x = var_567)[name = tensor("transpose_19")]; tensor var_572_shape = shape(x = key_15)[name = tensor("op_572_shape")]; tensor gather_29_indices_0 = const()[name = tensor("gather_29_indices_0"), val = tensor(2)]; tensor gather_29_axis_0 = const()[name = tensor("gather_29_axis_0"), val = tensor(0)]; tensor gather_29_batch_dims_0 = const()[name = tensor("gather_29_batch_dims_0"), val = tensor(0)]; tensor gather_29 = gather(axis = gather_29_axis_0, batch_dims = gather_29_batch_dims_0, indices = gather_29_indices_0, x = var_572_shape)[name = tensor("gather_29")]; tensor concat_41_values0_0 = const()[name = tensor("concat_41_values0_0"), val = tensor(0)]; tensor concat_41_values1_0 = const()[name = tensor("concat_41_values1_0"), val = tensor(0)]; tensor concat_41_values2_0 = const()[name = tensor("concat_41_values2_0"), val = tensor(0)]; tensor concat_41_axis_0 = const()[name = tensor("concat_41_axis_0"), val = tensor(0)]; tensor concat_41_interleave_0 = const()[name = tensor("concat_41_interleave_0"), val = tensor(false)]; tensor concat_41 = concat(axis = concat_41_axis_0, interleave = concat_41_interleave_0, values = (concat_41_values0_0, concat_41_values1_0, concat_41_values2_0, gather_29))[name = tensor("concat_41")]; tensor attention_mask_19_begin_0 = const()[name = tensor("attention_mask_19_begin_0"), val = tensor([0, 0, 0, 0])]; tensor attention_mask_19_end_mask_0 = const()[name = tensor("attention_mask_19_end_mask_0"), val = tensor([true, true, true, false])]; tensor attention_mask_19 = slice_by_index(begin = attention_mask_19_begin_0, end = concat_41, end_mask = attention_mask_19_end_mask_0, x = attention_mask_3)[name = tensor("attention_mask_19")]; tensor query_15 = transpose(perm = query_15_perm_0, x = var_558)[name = tensor("transpose_18")]; tensor mul_7 = mul(x = query_15, y = var_7)[name = tensor("mul_7")]; tensor matmul_7_transpose_y_0 = const()[name = tensor("matmul_7_transpose_y_0"), val = tensor(true)]; tensor matmul_7_transpose_x_0 = const()[name = tensor("matmul_7_transpose_x_0"), val = tensor(false)]; tensor matmul_7 = matmul(transpose_x = matmul_7_transpose_x_0, transpose_y = matmul_7_transpose_y_0, x = mul_7, y = key_15)[name = tensor("matmul_7")]; tensor add_7 = add(x = matmul_7, y = attention_mask_19)[name = tensor("add_7")]; tensor softmax_7_axis_0 = const()[name = tensor("softmax_7_axis_0"), val = tensor(-1)]; tensor softmax_7 = softmax(axis = softmax_7_axis_0, x = add_7)[name = tensor("softmax_7")]; tensor attn_output_29_transpose_x_0 = const()[name = tensor("attn_output_29_transpose_x_0"), val = tensor(false)]; tensor attn_output_29_transpose_y_0 = const()[name = tensor("attn_output_29_transpose_y_0"), val = tensor(false)]; tensor value_15 = transpose(perm = value_15_perm_0, x = var_570)[name = tensor("transpose_17")]; tensor attn_output_29 = matmul(transpose_x = attn_output_29_transpose_x_0, transpose_y = attn_output_29_transpose_y_0, x = softmax_7, y = value_15)[name = tensor("attn_output_29")]; tensor var_578_perm_0 = const()[name = tensor("op_578_perm_0"), val = tensor([0, 2, 1, 3])]; tensor concat_42_axis_0 = const()[name = tensor("concat_42_axis_0"), val = tensor(0)]; tensor concat_42_interleave_0 = const()[name = tensor("concat_42_interleave_0"), val = tensor(false)]; tensor concat_42 = concat(axis = concat_42_axis_0, interleave = concat_42_interleave_0, values = (gather_27, gather_28, var_23))[name = tensor("concat_42")]; tensor var_578 = transpose(perm = var_578_perm_0, x = attn_output_29)[name = tensor("transpose_16")]; tensor var_581 = reshape(shape = concat_42, x = var_578)[name = tensor("op_581")]; tensor input_135 = linear(bias = encoder_layers_7_self_attn_out_proj_bias, weight = encoder_layers_7_self_attn_out_proj_weight_palettized, x = var_581)[name = tensor("linear_45")]; tensor input_137 = add(x = input_131, y = input_135)[name = tensor("input_137")]; tensor input_139_axes_0 = const()[name = tensor("input_139_axes_0"), val = tensor([-1])]; tensor input_139 = layer_norm(axes = input_139_axes_0, beta = encoder_layers_7_final_layer_norm_bias, epsilon = var_5, gamma = encoder_layers_7_final_layer_norm_weight, x = input_137)[name = tensor("input_139")]; tensor input_141 = linear(bias = encoder_layers_7_fc1_bias, weight = encoder_layers_7_fc1_weight_palettized, x = input_139)[name = tensor("linear_46")]; tensor input_143 = relu(x = input_141)[name = tensor("input_143")]; tensor input_147 = linear(bias = encoder_layers_7_fc2_bias, weight = encoder_layers_7_fc2_weight_palettized, x = input_143)[name = tensor("linear_47")]; tensor input_149 = add(x = input_137, y = input_147)[name = tensor("input_149")]; tensor hidden_states_49_axes_0 = const()[name = tensor("hidden_states_49_axes_0"), val = tensor([-1])]; tensor hidden_states_49 = layer_norm(axes = hidden_states_49_axes_0, beta = encoder_layers_8_self_attn_layer_norm_bias, epsilon = var_5, gamma = encoder_layers_8_self_attn_layer_norm_weight, x = input_149)[name = tensor("hidden_states_49")]; tensor var_615_shape = shape(x = hidden_states_49)[name = tensor("op_615_shape")]; tensor gather_30 = const()[name = tensor("gather_30"), val = tensor(1)]; tensor gather_31_indices_0 = const()[name = tensor("gather_31_indices_0"), val = tensor(1)]; tensor gather_31_axis_0 = const()[name = tensor("gather_31_axis_0"), val = tensor(0)]; tensor gather_31_batch_dims_0 = const()[name = tensor("gather_31_batch_dims_0"), val = tensor(0)]; tensor gather_31 = gather(axis = gather_31_axis_0, batch_dims = gather_31_batch_dims_0, indices = gather_31_indices_0, x = var_615_shape)[name = tensor("gather_31")]; tensor var_619 = linear(bias = encoder_layers_8_self_attn_q_proj_bias, weight = encoder_layers_8_self_attn_q_proj_weight_palettized, x = hidden_states_49)[name = tensor("linear_48")]; tensor concat_43x = const()[name = tensor("concat_43x"), val = tensor([1, -1, 16, 64])]; tensor var_621 = reshape(shape = concat_43x, x = var_619)[name = tensor("op_621")]; tensor query_17_perm_0 = const()[name = tensor("query_17_perm_0"), val = tensor([0, 2, 1, 3])]; tensor key_states_17 = linear(bias = encoder_layers_8_self_attn_k_proj_bias, weight = encoder_layers_8_self_attn_k_proj_weight_palettized, x = hidden_states_49)[name = tensor("linear_49")]; tensor value_states_17 = linear(bias = encoder_layers_8_self_attn_v_proj_bias, weight = encoder_layers_8_self_attn_v_proj_weight_palettized, x = hidden_states_49)[name = tensor("linear_50")]; tensor concat_44x = const()[name = tensor("concat_44x"), val = tensor([1, -1, 16, 64])]; tensor var_630 = reshape(shape = concat_44x, x = key_states_17)[name = tensor("op_630")]; tensor key_17_perm_0 = const()[name = tensor("key_17_perm_0"), val = tensor([0, 2, -3, -1])]; tensor concat_45x = const()[name = tensor("concat_45x"), val = tensor([1, -1, 16, 64])]; tensor var_633 = reshape(shape = concat_45x, x = value_states_17)[name = tensor("op_633")]; tensor value_17_perm_0 = const()[name = tensor("value_17_perm_0"), val = tensor([0, 2, -3, -1])]; tensor key_17 = transpose(perm = key_17_perm_0, x = var_630)[name = tensor("transpose_15")]; tensor var_635_shape = shape(x = key_17)[name = tensor("op_635_shape")]; tensor gather_32_indices_0 = const()[name = tensor("gather_32_indices_0"), val = tensor(2)]; tensor gather_32_axis_0 = const()[name = tensor("gather_32_axis_0"), val = tensor(0)]; tensor gather_32_batch_dims_0 = const()[name = tensor("gather_32_batch_dims_0"), val = tensor(0)]; tensor gather_32 = gather(axis = gather_32_axis_0, batch_dims = gather_32_batch_dims_0, indices = gather_32_indices_0, x = var_635_shape)[name = tensor("gather_32")]; tensor concat_46_values0_0 = const()[name = tensor("concat_46_values0_0"), val = tensor(0)]; tensor concat_46_values1_0 = const()[name = tensor("concat_46_values1_0"), val = tensor(0)]; tensor concat_46_values2_0 = const()[name = tensor("concat_46_values2_0"), val = tensor(0)]; tensor concat_46_axis_0 = const()[name = tensor("concat_46_axis_0"), val = tensor(0)]; tensor concat_46_interleave_0 = const()[name = tensor("concat_46_interleave_0"), val = tensor(false)]; tensor concat_46 = concat(axis = concat_46_axis_0, interleave = concat_46_interleave_0, values = (concat_46_values0_0, concat_46_values1_0, concat_46_values2_0, gather_32))[name = tensor("concat_46")]; tensor attention_mask_21_begin_0 = const()[name = tensor("attention_mask_21_begin_0"), val = tensor([0, 0, 0, 0])]; tensor attention_mask_21_end_mask_0 = const()[name = tensor("attention_mask_21_end_mask_0"), val = tensor([true, true, true, false])]; tensor attention_mask_21 = slice_by_index(begin = attention_mask_21_begin_0, end = concat_46, end_mask = attention_mask_21_end_mask_0, x = attention_mask_3)[name = tensor("attention_mask_21")]; tensor query_17 = transpose(perm = query_17_perm_0, x = var_621)[name = tensor("transpose_14")]; tensor mul_8 = mul(x = query_17, y = var_7)[name = tensor("mul_8")]; tensor matmul_8_transpose_y_0 = const()[name = tensor("matmul_8_transpose_y_0"), val = tensor(true)]; tensor matmul_8_transpose_x_0 = const()[name = tensor("matmul_8_transpose_x_0"), val = tensor(false)]; tensor matmul_8 = matmul(transpose_x = matmul_8_transpose_x_0, transpose_y = matmul_8_transpose_y_0, x = mul_8, y = key_17)[name = tensor("matmul_8")]; tensor add_8 = add(x = matmul_8, y = attention_mask_21)[name = tensor("add_8")]; tensor softmax_8_axis_0 = const()[name = tensor("softmax_8_axis_0"), val = tensor(-1)]; tensor softmax_8 = softmax(axis = softmax_8_axis_0, x = add_8)[name = tensor("softmax_8")]; tensor attn_output_33_transpose_x_0 = const()[name = tensor("attn_output_33_transpose_x_0"), val = tensor(false)]; tensor attn_output_33_transpose_y_0 = const()[name = tensor("attn_output_33_transpose_y_0"), val = tensor(false)]; tensor value_17 = transpose(perm = value_17_perm_0, x = var_633)[name = tensor("transpose_13")]; tensor attn_output_33 = matmul(transpose_x = attn_output_33_transpose_x_0, transpose_y = attn_output_33_transpose_y_0, x = softmax_8, y = value_17)[name = tensor("attn_output_33")]; tensor var_641_perm_0 = const()[name = tensor("op_641_perm_0"), val = tensor([0, 2, 1, 3])]; tensor concat_47_axis_0 = const()[name = tensor("concat_47_axis_0"), val = tensor(0)]; tensor concat_47_interleave_0 = const()[name = tensor("concat_47_interleave_0"), val = tensor(false)]; tensor concat_47 = concat(axis = concat_47_axis_0, interleave = concat_47_interleave_0, values = (gather_30, gather_31, var_23))[name = tensor("concat_47")]; tensor var_641 = transpose(perm = var_641_perm_0, x = attn_output_33)[name = tensor("transpose_12")]; tensor var_644 = reshape(shape = concat_47, x = var_641)[name = tensor("op_644")]; tensor input_153 = linear(bias = encoder_layers_8_self_attn_out_proj_bias, weight = encoder_layers_8_self_attn_out_proj_weight_palettized, x = var_644)[name = tensor("linear_51")]; tensor input_155 = add(x = input_149, y = input_153)[name = tensor("input_155")]; tensor input_157_axes_0 = const()[name = tensor("input_157_axes_0"), val = tensor([-1])]; tensor input_157 = layer_norm(axes = input_157_axes_0, beta = encoder_layers_8_final_layer_norm_bias, epsilon = var_5, gamma = encoder_layers_8_final_layer_norm_weight, x = input_155)[name = tensor("input_157")]; tensor input_159 = linear(bias = encoder_layers_8_fc1_bias, weight = encoder_layers_8_fc1_weight_palettized, x = input_157)[name = tensor("linear_52")]; tensor input_161 = relu(x = input_159)[name = tensor("input_161")]; tensor input_165 = linear(bias = encoder_layers_8_fc2_bias, weight = encoder_layers_8_fc2_weight_palettized, x = input_161)[name = tensor("linear_53")]; tensor input_167 = add(x = input_155, y = input_165)[name = tensor("input_167")]; tensor hidden_states_55_axes_0 = const()[name = tensor("hidden_states_55_axes_0"), val = tensor([-1])]; tensor hidden_states_55 = layer_norm(axes = hidden_states_55_axes_0, beta = encoder_layers_9_self_attn_layer_norm_bias, epsilon = var_5, gamma = encoder_layers_9_self_attn_layer_norm_weight, x = input_167)[name = tensor("hidden_states_55")]; tensor var_678_shape = shape(x = hidden_states_55)[name = tensor("op_678_shape")]; tensor gather_33 = const()[name = tensor("gather_33"), val = tensor(1)]; tensor gather_34_indices_0 = const()[name = tensor("gather_34_indices_0"), val = tensor(1)]; tensor gather_34_axis_0 = const()[name = tensor("gather_34_axis_0"), val = tensor(0)]; tensor gather_34_batch_dims_0 = const()[name = tensor("gather_34_batch_dims_0"), val = tensor(0)]; tensor gather_34 = gather(axis = gather_34_axis_0, batch_dims = gather_34_batch_dims_0, indices = gather_34_indices_0, x = var_678_shape)[name = tensor("gather_34")]; tensor var_682 = linear(bias = encoder_layers_9_self_attn_q_proj_bias, weight = encoder_layers_9_self_attn_q_proj_weight_palettized, x = hidden_states_55)[name = tensor("linear_54")]; tensor concat_48x = const()[name = tensor("concat_48x"), val = tensor([1, -1, 16, 64])]; tensor var_684 = reshape(shape = concat_48x, x = var_682)[name = tensor("op_684")]; tensor query_19_perm_0 = const()[name = tensor("query_19_perm_0"), val = tensor([0, 2, 1, 3])]; tensor key_states_19 = linear(bias = encoder_layers_9_self_attn_k_proj_bias, weight = encoder_layers_9_self_attn_k_proj_weight_palettized, x = hidden_states_55)[name = tensor("linear_55")]; tensor value_states_19 = linear(bias = encoder_layers_9_self_attn_v_proj_bias, weight = encoder_layers_9_self_attn_v_proj_weight_palettized, x = hidden_states_55)[name = tensor("linear_56")]; tensor concat_49x = const()[name = tensor("concat_49x"), val = tensor([1, -1, 16, 64])]; tensor var_693 = reshape(shape = concat_49x, x = key_states_19)[name = tensor("op_693")]; tensor key_19_perm_0 = const()[name = tensor("key_19_perm_0"), val = tensor([0, 2, -3, -1])]; tensor concat_50x = const()[name = tensor("concat_50x"), val = tensor([1, -1, 16, 64])]; tensor var_696 = reshape(shape = concat_50x, x = value_states_19)[name = tensor("op_696")]; tensor value_19_perm_0 = const()[name = tensor("value_19_perm_0"), val = tensor([0, 2, -3, -1])]; tensor key_19 = transpose(perm = key_19_perm_0, x = var_693)[name = tensor("transpose_11")]; tensor var_698_shape = shape(x = key_19)[name = tensor("op_698_shape")]; tensor gather_35_indices_0 = const()[name = tensor("gather_35_indices_0"), val = tensor(2)]; tensor gather_35_axis_0 = const()[name = tensor("gather_35_axis_0"), val = tensor(0)]; tensor gather_35_batch_dims_0 = const()[name = tensor("gather_35_batch_dims_0"), val = tensor(0)]; tensor gather_35 = gather(axis = gather_35_axis_0, batch_dims = gather_35_batch_dims_0, indices = gather_35_indices_0, x = var_698_shape)[name = tensor("gather_35")]; tensor concat_51_values0_0 = const()[name = tensor("concat_51_values0_0"), val = tensor(0)]; tensor concat_51_values1_0 = const()[name = tensor("concat_51_values1_0"), val = tensor(0)]; tensor concat_51_values2_0 = const()[name = tensor("concat_51_values2_0"), val = tensor(0)]; tensor concat_51_axis_0 = const()[name = tensor("concat_51_axis_0"), val = tensor(0)]; tensor concat_51_interleave_0 = const()[name = tensor("concat_51_interleave_0"), val = tensor(false)]; tensor concat_51 = concat(axis = concat_51_axis_0, interleave = concat_51_interleave_0, values = (concat_51_values0_0, concat_51_values1_0, concat_51_values2_0, gather_35))[name = tensor("concat_51")]; tensor attention_mask_23_begin_0 = const()[name = tensor("attention_mask_23_begin_0"), val = tensor([0, 0, 0, 0])]; tensor attention_mask_23_end_mask_0 = const()[name = tensor("attention_mask_23_end_mask_0"), val = tensor([true, true, true, false])]; tensor attention_mask_23 = slice_by_index(begin = attention_mask_23_begin_0, end = concat_51, end_mask = attention_mask_23_end_mask_0, x = attention_mask_3)[name = tensor("attention_mask_23")]; tensor query_19 = transpose(perm = query_19_perm_0, x = var_684)[name = tensor("transpose_10")]; tensor mul_9 = mul(x = query_19, y = var_7)[name = tensor("mul_9")]; tensor matmul_9_transpose_y_0 = const()[name = tensor("matmul_9_transpose_y_0"), val = tensor(true)]; tensor matmul_9_transpose_x_0 = const()[name = tensor("matmul_9_transpose_x_0"), val = tensor(false)]; tensor matmul_9 = matmul(transpose_x = matmul_9_transpose_x_0, transpose_y = matmul_9_transpose_y_0, x = mul_9, y = key_19)[name = tensor("matmul_9")]; tensor add_9 = add(x = matmul_9, y = attention_mask_23)[name = tensor("add_9")]; tensor softmax_9_axis_0 = const()[name = tensor("softmax_9_axis_0"), val = tensor(-1)]; tensor softmax_9 = softmax(axis = softmax_9_axis_0, x = add_9)[name = tensor("softmax_9")]; tensor attn_output_37_transpose_x_0 = const()[name = tensor("attn_output_37_transpose_x_0"), val = tensor(false)]; tensor attn_output_37_transpose_y_0 = const()[name = tensor("attn_output_37_transpose_y_0"), val = tensor(false)]; tensor value_19 = transpose(perm = value_19_perm_0, x = var_696)[name = tensor("transpose_9")]; tensor attn_output_37 = matmul(transpose_x = attn_output_37_transpose_x_0, transpose_y = attn_output_37_transpose_y_0, x = softmax_9, y = value_19)[name = tensor("attn_output_37")]; tensor var_704_perm_0 = const()[name = tensor("op_704_perm_0"), val = tensor([0, 2, 1, 3])]; tensor concat_52_axis_0 = const()[name = tensor("concat_52_axis_0"), val = tensor(0)]; tensor concat_52_interleave_0 = const()[name = tensor("concat_52_interleave_0"), val = tensor(false)]; tensor concat_52 = concat(axis = concat_52_axis_0, interleave = concat_52_interleave_0, values = (gather_33, gather_34, var_23))[name = tensor("concat_52")]; tensor var_704 = transpose(perm = var_704_perm_0, x = attn_output_37)[name = tensor("transpose_8")]; tensor var_707 = reshape(shape = concat_52, x = var_704)[name = tensor("op_707")]; tensor input_171 = linear(bias = encoder_layers_9_self_attn_out_proj_bias, weight = encoder_layers_9_self_attn_out_proj_weight_palettized, x = var_707)[name = tensor("linear_57")]; tensor input_173 = add(x = input_167, y = input_171)[name = tensor("input_173")]; tensor input_175_axes_0 = const()[name = tensor("input_175_axes_0"), val = tensor([-1])]; tensor input_175 = layer_norm(axes = input_175_axes_0, beta = encoder_layers_9_final_layer_norm_bias, epsilon = var_5, gamma = encoder_layers_9_final_layer_norm_weight, x = input_173)[name = tensor("input_175")]; tensor input_177 = linear(bias = encoder_layers_9_fc1_bias, weight = encoder_layers_9_fc1_weight_palettized, x = input_175)[name = tensor("linear_58")]; tensor input_179 = relu(x = input_177)[name = tensor("input_179")]; tensor input_183 = linear(bias = encoder_layers_9_fc2_bias, weight = encoder_layers_9_fc2_weight_palettized, x = input_179)[name = tensor("linear_59")]; tensor input_185 = add(x = input_173, y = input_183)[name = tensor("input_185")]; tensor hidden_states_61_axes_0 = const()[name = tensor("hidden_states_61_axes_0"), val = tensor([-1])]; tensor hidden_states_61 = layer_norm(axes = hidden_states_61_axes_0, beta = encoder_layers_10_self_attn_layer_norm_bias, epsilon = var_5, gamma = encoder_layers_10_self_attn_layer_norm_weight, x = input_185)[name = tensor("hidden_states_61")]; tensor var_741_shape = shape(x = hidden_states_61)[name = tensor("op_741_shape")]; tensor gather_36 = const()[name = tensor("gather_36"), val = tensor(1)]; tensor gather_37_indices_0 = const()[name = tensor("gather_37_indices_0"), val = tensor(1)]; tensor gather_37_axis_0 = const()[name = tensor("gather_37_axis_0"), val = tensor(0)]; tensor gather_37_batch_dims_0 = const()[name = tensor("gather_37_batch_dims_0"), val = tensor(0)]; tensor gather_37 = gather(axis = gather_37_axis_0, batch_dims = gather_37_batch_dims_0, indices = gather_37_indices_0, x = var_741_shape)[name = tensor("gather_37")]; tensor var_745 = linear(bias = encoder_layers_10_self_attn_q_proj_bias, weight = encoder_layers_10_self_attn_q_proj_weight_palettized, x = hidden_states_61)[name = tensor("linear_60")]; tensor concat_53x = const()[name = tensor("concat_53x"), val = tensor([1, -1, 16, 64])]; tensor var_747 = reshape(shape = concat_53x, x = var_745)[name = tensor("op_747")]; tensor query_21_perm_0 = const()[name = tensor("query_21_perm_0"), val = tensor([0, 2, 1, 3])]; tensor key_states_21 = linear(bias = encoder_layers_10_self_attn_k_proj_bias, weight = encoder_layers_10_self_attn_k_proj_weight_palettized, x = hidden_states_61)[name = tensor("linear_61")]; tensor value_states_21 = linear(bias = encoder_layers_10_self_attn_v_proj_bias, weight = encoder_layers_10_self_attn_v_proj_weight_palettized, x = hidden_states_61)[name = tensor("linear_62")]; tensor concat_54x = const()[name = tensor("concat_54x"), val = tensor([1, -1, 16, 64])]; tensor var_756 = reshape(shape = concat_54x, x = key_states_21)[name = tensor("op_756")]; tensor key_21_perm_0 = const()[name = tensor("key_21_perm_0"), val = tensor([0, 2, -3, -1])]; tensor concat_55x = const()[name = tensor("concat_55x"), val = tensor([1, -1, 16, 64])]; tensor var_759 = reshape(shape = concat_55x, x = value_states_21)[name = tensor("op_759")]; tensor value_21_perm_0 = const()[name = tensor("value_21_perm_0"), val = tensor([0, 2, -3, -1])]; tensor key_21 = transpose(perm = key_21_perm_0, x = var_756)[name = tensor("transpose_7")]; tensor var_761_shape = shape(x = key_21)[name = tensor("op_761_shape")]; tensor gather_38_indices_0 = const()[name = tensor("gather_38_indices_0"), val = tensor(2)]; tensor gather_38_axis_0 = const()[name = tensor("gather_38_axis_0"), val = tensor(0)]; tensor gather_38_batch_dims_0 = const()[name = tensor("gather_38_batch_dims_0"), val = tensor(0)]; tensor gather_38 = gather(axis = gather_38_axis_0, batch_dims = gather_38_batch_dims_0, indices = gather_38_indices_0, x = var_761_shape)[name = tensor("gather_38")]; tensor concat_56_values0_0 = const()[name = tensor("concat_56_values0_0"), val = tensor(0)]; tensor concat_56_values1_0 = const()[name = tensor("concat_56_values1_0"), val = tensor(0)]; tensor concat_56_values2_0 = const()[name = tensor("concat_56_values2_0"), val = tensor(0)]; tensor concat_56_axis_0 = const()[name = tensor("concat_56_axis_0"), val = tensor(0)]; tensor concat_56_interleave_0 = const()[name = tensor("concat_56_interleave_0"), val = tensor(false)]; tensor concat_56 = concat(axis = concat_56_axis_0, interleave = concat_56_interleave_0, values = (concat_56_values0_0, concat_56_values1_0, concat_56_values2_0, gather_38))[name = tensor("concat_56")]; tensor attention_mask_25_begin_0 = const()[name = tensor("attention_mask_25_begin_0"), val = tensor([0, 0, 0, 0])]; tensor attention_mask_25_end_mask_0 = const()[name = tensor("attention_mask_25_end_mask_0"), val = tensor([true, true, true, false])]; tensor attention_mask_25 = slice_by_index(begin = attention_mask_25_begin_0, end = concat_56, end_mask = attention_mask_25_end_mask_0, x = attention_mask_3)[name = tensor("attention_mask_25")]; tensor query_21 = transpose(perm = query_21_perm_0, x = var_747)[name = tensor("transpose_6")]; tensor mul_10 = mul(x = query_21, y = var_7)[name = tensor("mul_10")]; tensor matmul_10_transpose_y_0 = const()[name = tensor("matmul_10_transpose_y_0"), val = tensor(true)]; tensor matmul_10_transpose_x_0 = const()[name = tensor("matmul_10_transpose_x_0"), val = tensor(false)]; tensor matmul_10 = matmul(transpose_x = matmul_10_transpose_x_0, transpose_y = matmul_10_transpose_y_0, x = mul_10, y = key_21)[name = tensor("matmul_10")]; tensor add_10 = add(x = matmul_10, y = attention_mask_25)[name = tensor("add_10")]; tensor softmax_10_axis_0 = const()[name = tensor("softmax_10_axis_0"), val = tensor(-1)]; tensor softmax_10 = softmax(axis = softmax_10_axis_0, x = add_10)[name = tensor("softmax_10")]; tensor attn_output_41_transpose_x_0 = const()[name = tensor("attn_output_41_transpose_x_0"), val = tensor(false)]; tensor attn_output_41_transpose_y_0 = const()[name = tensor("attn_output_41_transpose_y_0"), val = tensor(false)]; tensor value_21 = transpose(perm = value_21_perm_0, x = var_759)[name = tensor("transpose_5")]; tensor attn_output_41 = matmul(transpose_x = attn_output_41_transpose_x_0, transpose_y = attn_output_41_transpose_y_0, x = softmax_10, y = value_21)[name = tensor("attn_output_41")]; tensor var_767_perm_0 = const()[name = tensor("op_767_perm_0"), val = tensor([0, 2, 1, 3])]; tensor concat_57_axis_0 = const()[name = tensor("concat_57_axis_0"), val = tensor(0)]; tensor concat_57_interleave_0 = const()[name = tensor("concat_57_interleave_0"), val = tensor(false)]; tensor concat_57 = concat(axis = concat_57_axis_0, interleave = concat_57_interleave_0, values = (gather_36, gather_37, var_23))[name = tensor("concat_57")]; tensor var_767 = transpose(perm = var_767_perm_0, x = attn_output_41)[name = tensor("transpose_4")]; tensor var_770 = reshape(shape = concat_57, x = var_767)[name = tensor("op_770")]; tensor input_189 = linear(bias = encoder_layers_10_self_attn_out_proj_bias, weight = encoder_layers_10_self_attn_out_proj_weight_palettized, x = var_770)[name = tensor("linear_63")]; tensor input_191 = add(x = input_185, y = input_189)[name = tensor("input_191")]; tensor input_193_axes_0 = const()[name = tensor("input_193_axes_0"), val = tensor([-1])]; tensor input_193 = layer_norm(axes = input_193_axes_0, beta = encoder_layers_10_final_layer_norm_bias, epsilon = var_5, gamma = encoder_layers_10_final_layer_norm_weight, x = input_191)[name = tensor("input_193")]; tensor input_195 = linear(bias = encoder_layers_10_fc1_bias, weight = encoder_layers_10_fc1_weight_palettized, x = input_193)[name = tensor("linear_64")]; tensor input_197 = relu(x = input_195)[name = tensor("input_197")]; tensor input_201 = linear(bias = encoder_layers_10_fc2_bias, weight = encoder_layers_10_fc2_weight_palettized, x = input_197)[name = tensor("linear_65")]; tensor input_203 = add(x = input_191, y = input_201)[name = tensor("input_203")]; tensor hidden_states_67_axes_0 = const()[name = tensor("hidden_states_67_axes_0"), val = tensor([-1])]; tensor hidden_states_67 = layer_norm(axes = hidden_states_67_axes_0, beta = encoder_layers_11_self_attn_layer_norm_bias, epsilon = var_5, gamma = encoder_layers_11_self_attn_layer_norm_weight, x = input_203)[name = tensor("hidden_states_67")]; tensor var_804_shape = shape(x = hidden_states_67)[name = tensor("op_804_shape")]; tensor gather_39 = const()[name = tensor("gather_39"), val = tensor(1)]; tensor gather_40_indices_0 = const()[name = tensor("gather_40_indices_0"), val = tensor(1)]; tensor gather_40_axis_0 = const()[name = tensor("gather_40_axis_0"), val = tensor(0)]; tensor gather_40_batch_dims_0 = const()[name = tensor("gather_40_batch_dims_0"), val = tensor(0)]; tensor gather_40 = gather(axis = gather_40_axis_0, batch_dims = gather_40_batch_dims_0, indices = gather_40_indices_0, x = var_804_shape)[name = tensor("gather_40")]; tensor var_808 = linear(bias = encoder_layers_11_self_attn_q_proj_bias, weight = encoder_layers_11_self_attn_q_proj_weight_palettized, x = hidden_states_67)[name = tensor("linear_66")]; tensor concat_58x = const()[name = tensor("concat_58x"), val = tensor([1, -1, 16, 64])]; tensor var_810 = reshape(shape = concat_58x, x = var_808)[name = tensor("op_810")]; tensor query_perm_0 = const()[name = tensor("query_perm_0"), val = tensor([0, 2, 1, 3])]; tensor key_states = linear(bias = encoder_layers_11_self_attn_k_proj_bias, weight = encoder_layers_11_self_attn_k_proj_weight_palettized, x = hidden_states_67)[name = tensor("linear_67")]; tensor value_states = linear(bias = encoder_layers_11_self_attn_v_proj_bias, weight = encoder_layers_11_self_attn_v_proj_weight_palettized, x = hidden_states_67)[name = tensor("linear_68")]; tensor concat_59x = const()[name = tensor("concat_59x"), val = tensor([1, -1, 16, 64])]; tensor var_819 = reshape(shape = concat_59x, x = key_states)[name = tensor("op_819")]; tensor key_perm_0 = const()[name = tensor("key_perm_0"), val = tensor([0, 2, -3, -1])]; tensor concat_60x = const()[name = tensor("concat_60x"), val = tensor([1, -1, 16, 64])]; tensor var_822 = reshape(shape = concat_60x, x = value_states)[name = tensor("op_822")]; tensor value_perm_0 = const()[name = tensor("value_perm_0"), val = tensor([0, 2, -3, -1])]; tensor key = transpose(perm = key_perm_0, x = var_819)[name = tensor("transpose_3")]; tensor var_824_shape = shape(x = key)[name = tensor("op_824_shape")]; tensor gather_41_indices_0 = const()[name = tensor("gather_41_indices_0"), val = tensor(2)]; tensor gather_41_axis_0 = const()[name = tensor("gather_41_axis_0"), val = tensor(0)]; tensor gather_41_batch_dims_0 = const()[name = tensor("gather_41_batch_dims_0"), val = tensor(0)]; tensor gather_41 = gather(axis = gather_41_axis_0, batch_dims = gather_41_batch_dims_0, indices = gather_41_indices_0, x = var_824_shape)[name = tensor("gather_41")]; tensor concat_61_values0_0 = const()[name = tensor("concat_61_values0_0"), val = tensor(0)]; tensor concat_61_values1_0 = const()[name = tensor("concat_61_values1_0"), val = tensor(0)]; tensor concat_61_values2_0 = const()[name = tensor("concat_61_values2_0"), val = tensor(0)]; tensor concat_61_axis_0 = const()[name = tensor("concat_61_axis_0"), val = tensor(0)]; tensor concat_61_interleave_0 = const()[name = tensor("concat_61_interleave_0"), val = tensor(false)]; tensor concat_61 = concat(axis = concat_61_axis_0, interleave = concat_61_interleave_0, values = (concat_61_values0_0, concat_61_values1_0, concat_61_values2_0, gather_41))[name = tensor("concat_61")]; tensor attention_mask_begin_0 = const()[name = tensor("attention_mask_begin_0"), val = tensor([0, 0, 0, 0])]; tensor attention_mask_end_mask_0 = const()[name = tensor("attention_mask_end_mask_0"), val = tensor([true, true, true, false])]; tensor attention_mask_1 = slice_by_index(begin = attention_mask_begin_0, end = concat_61, end_mask = attention_mask_end_mask_0, x = attention_mask_3)[name = tensor("attention_mask")]; tensor query = transpose(perm = query_perm_0, x = var_810)[name = tensor("transpose_2")]; tensor mul_11 = mul(x = query, y = var_7)[name = tensor("mul_11")]; tensor matmul_11_transpose_y_0 = const()[name = tensor("matmul_11_transpose_y_0"), val = tensor(true)]; tensor matmul_11_transpose_x_0 = const()[name = tensor("matmul_11_transpose_x_0"), val = tensor(false)]; tensor matmul_11 = matmul(transpose_x = matmul_11_transpose_x_0, transpose_y = matmul_11_transpose_y_0, x = mul_11, y = key)[name = tensor("matmul_11")]; tensor add_11 = add(x = matmul_11, y = attention_mask_1)[name = tensor("add_11")]; tensor softmax_11_axis_0 = const()[name = tensor("softmax_11_axis_0"), val = tensor(-1)]; tensor softmax_11 = softmax(axis = softmax_11_axis_0, x = add_11)[name = tensor("softmax_11")]; tensor attn_output_45_transpose_x_0 = const()[name = tensor("attn_output_45_transpose_x_0"), val = tensor(false)]; tensor attn_output_45_transpose_y_0 = const()[name = tensor("attn_output_45_transpose_y_0"), val = tensor(false)]; tensor value = transpose(perm = value_perm_0, x = var_822)[name = tensor("transpose_1")]; tensor attn_output_45 = matmul(transpose_x = attn_output_45_transpose_x_0, transpose_y = attn_output_45_transpose_y_0, x = softmax_11, y = value)[name = tensor("attn_output_45")]; tensor var_830_perm_0 = const()[name = tensor("op_830_perm_0"), val = tensor([0, 2, 1, 3])]; tensor concat_62_axis_0 = const()[name = tensor("concat_62_axis_0"), val = tensor(0)]; tensor concat_62_interleave_0 = const()[name = tensor("concat_62_interleave_0"), val = tensor(false)]; tensor concat_62 = concat(axis = concat_62_axis_0, interleave = concat_62_interleave_0, values = (gather_39, gather_40, var_23))[name = tensor("concat_62")]; tensor var_830 = transpose(perm = var_830_perm_0, x = attn_output_45)[name = tensor("transpose_0")]; tensor var_833 = reshape(shape = concat_62, x = var_830)[name = tensor("op_833")]; tensor input_207 = linear(bias = encoder_layers_11_self_attn_out_proj_bias, weight = encoder_layers_11_self_attn_out_proj_weight_palettized, x = var_833)[name = tensor("linear_69")]; tensor input_209 = add(x = input_203, y = input_207)[name = tensor("input_209")]; tensor input_211_axes_0 = const()[name = tensor("input_211_axes_0"), val = tensor([-1])]; tensor input_211 = layer_norm(axes = input_211_axes_0, beta = encoder_layers_11_final_layer_norm_bias, epsilon = var_5, gamma = encoder_layers_11_final_layer_norm_weight, x = input_209)[name = tensor("input_211")]; tensor input_213 = linear(bias = encoder_layers_11_fc1_bias, weight = encoder_layers_11_fc1_weight_palettized, x = input_211)[name = tensor("linear_70")]; tensor input_215 = relu(x = input_213)[name = tensor("input_215")]; tensor input_219 = linear(bias = encoder_layers_11_fc2_bias, weight = encoder_layers_11_fc2_weight_palettized, x = input_215)[name = tensor("linear_71")]; tensor input = add(x = input_209, y = input_219)[name = tensor("input")]; tensor var_857_axes_0 = const()[name = tensor("op_857_axes_0"), val = tensor([-1])]; tensor output = layer_norm(axes = var_857_axes_0, beta = encoder_layer_norm_bias, epsilon = var_5, gamma = encoder_layer_norm_weight, x = input)[name = tensor("op_857")]; } -> (output); }