diff --git "a/text/L1024-gptq/CuaS1Decoder_part1.mlmodelc/model.mil" "b/text/L1024-gptq/CuaS1Decoder_part1.mlmodelc/model.mil"
new file mode 100644--- /dev/null
+++ "b/text/L1024-gptq/CuaS1Decoder_part1.mlmodelc/model.mil"
@@ -0,0 +1,7569 @@
+program(1.3)
+[buildInfo = dict<string, string>({{"coremlc-component-MIL", "3600.16.1"}, {"coremlc-version", "3600.25.2"}, {"coremltools-component-torch", "2.7.0"}, {"coremltools-source-dialect", "TorchScript"}, {"coremltools-version", "9.0"}})]
+{
+    func main<ios18>(tensor<fp16, [1024, 64]> cos, tensor<fp16, [1, 1024, 2560]> hidden, tensor<fp16, [1024, 64]> sin) {
+            tensor<fp16, [8192, 2560]> layers_0_linear_attn_in_proj_qkv_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [8192, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(64))), offset = tensor<uint8, [8192, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(20988096))), scale = tensor<fp16, [8192, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(20971648))))[name = string("layers_0_linear_attn_in_proj_qkv_weight_cast_fp16")];
+            tensor<fp16, [4096, 2560]> layers_0_linear_attn_in_proj_z_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [4096, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(20996352))), offset = tensor<uint8, [4096, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(31490432))), scale = tensor<fp16, [4096, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(31482176))))[name = string("layers_0_linear_attn_in_proj_z_weight_cast_fp16")];
+            tensor<fp16, [32, 2560]> layers_0_linear_attn_in_proj_b_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [32, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(31494592))), offset = tensor<uint8, [32, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(31576704))), scale = tensor<fp16, [32, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(31576576))))[name = string("layers_0_linear_attn_in_proj_b_weight_cast_fp16")];
+            tensor<fp16, [32, 2560]> layers_0_linear_attn_in_proj_a_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [32, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(31576832))), offset = tensor<uint8, [32, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(31658944))), scale = tensor<fp16, [32, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(31658816))))[name = string("layers_0_linear_attn_in_proj_a_weight_cast_fp16")];
+            tensor<fp16, [2560, 4096]> layers_0_linear_attn_out_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [2560, 4096]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(31659072))), offset = tensor<uint8, [2560, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(42150080))), scale = tensor<fp16, [2560, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(42144896))))[name = string("layers_0_linear_attn_out_proj_weight_cast_fp16")];
+            tensor<fp16, [9216, 2560]> layers_0_mlp_gate_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint4, [9216, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(42152704))), offset = tensor<uint4, [9216, 160]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(56898432))), scale = tensor<fp16, [9216, 160]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(53949248))))[name = string("layers_0_mlp_gate_proj_weight_cast_fp16")];
+            tensor<fp16, [9216, 2560]> layers_0_mlp_up_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint4, [9216, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(57635776))), offset = tensor<uint4, [9216, 160]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(56898432))), scale = tensor<fp16, [9216, 160]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(69432320))))[name = string("layers_0_mlp_up_proj_weight_cast_fp16")];
+            tensor<fp16, [2560, 9216]> layers_0_mlp_down_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint4, [2560, 9216]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(72381504))), offset = tensor<uint4, [2560, 576]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(87127232))), scale = tensor<fp16, [2560, 576]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(84178048))))[name = string("layers_0_mlp_down_proj_weight_cast_fp16")];
+            tensor<fp16, [8192, 2560]> layers_1_linear_attn_in_proj_qkv_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [8192, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(87864576))), offset = tensor<uint8, [8192, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(20988096))), scale = tensor<fp16, [8192, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(108836160))))[name = string("layers_1_linear_attn_in_proj_qkv_weight_cast_fp16")];
+            tensor<fp16, [4096, 2560]> layers_1_linear_attn_in_proj_z_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [4096, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(108852608))), offset = tensor<uint8, [4096, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(31490432))), scale = tensor<fp16, [4096, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(119338432))))[name = string("layers_1_linear_attn_in_proj_z_weight_cast_fp16")];
+            tensor<fp16, [32, 2560]> layers_1_linear_attn_in_proj_b_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [32, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(119346688))), offset = tensor<uint8, [32, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(119428800))), scale = tensor<fp16, [32, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(119428672))))[name = string("layers_1_linear_attn_in_proj_b_weight_cast_fp16")];
+            tensor<fp16, [32, 2560]> layers_1_linear_attn_in_proj_a_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [32, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(119428928))), offset = tensor<uint8, [32, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(119511040))), scale = tensor<fp16, [32, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(119510912))))[name = string("layers_1_linear_attn_in_proj_a_weight_cast_fp16")];
+            tensor<fp16, [2560, 4096]> layers_1_linear_attn_out_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [2560, 4096]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(119511168))), offset = tensor<uint8, [2560, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(42150080))), scale = tensor<fp16, [2560, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(129996992))))[name = string("layers_1_linear_attn_out_proj_weight_cast_fp16")];
+            tensor<fp16, [9216, 2560]> layers_1_mlp_gate_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint4, [9216, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(130002176))), offset = tensor<uint4, [9216, 160]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(56898432))), scale = tensor<fp16, [9216, 160]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(141798720))))[name = string("layers_1_mlp_gate_proj_weight_cast_fp16")];
+            tensor<fp16, [9216, 2560]> layers_1_mlp_up_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint4, [9216, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(144747904))), offset = tensor<uint4, [9216, 160]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(56898432))), scale = tensor<fp16, [9216, 160]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(156544448))))[name = string("layers_1_mlp_up_proj_weight_cast_fp16")];
+            tensor<fp16, [2560, 9216]> layers_1_mlp_down_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint4, [2560, 9216]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(159493632))), offset = tensor<uint4, [2560, 576]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(87127232))), scale = tensor<fp16, [2560, 576]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(171290176))))[name = string("layers_1_mlp_down_proj_weight_cast_fp16")];
+            tensor<fp16, [8192, 2560]> layers_2_linear_attn_in_proj_qkv_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [8192, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(174239360))), offset = tensor<uint8, [8192, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(20988096))), scale = tensor<fp16, [8192, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(195210944))))[name = string("layers_2_linear_attn_in_proj_qkv_weight_cast_fp16")];
+            tensor<fp16, [4096, 2560]> layers_2_linear_attn_in_proj_z_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [4096, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(195227392))), offset = tensor<uint8, [4096, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(31490432))), scale = tensor<fp16, [4096, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(205713216))))[name = string("layers_2_linear_attn_in_proj_z_weight_cast_fp16")];
+            tensor<fp16, [32, 2560]> layers_2_linear_attn_in_proj_b_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [32, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(205721472))), offset = tensor<uint8, [32, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(205803584))), scale = tensor<fp16, [32, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(205803456))))[name = string("layers_2_linear_attn_in_proj_b_weight_cast_fp16")];
+            tensor<fp16, [32, 2560]> layers_2_linear_attn_in_proj_a_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [32, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(205803712))), offset = tensor<uint8, [32, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(205885824))), scale = tensor<fp16, [32, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(205885696))))[name = string("layers_2_linear_attn_in_proj_a_weight_cast_fp16")];
+            tensor<fp16, [2560, 4096]> layers_2_linear_attn_out_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [2560, 4096]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(205885952))), offset = tensor<uint8, [2560, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(42150080))), scale = tensor<fp16, [2560, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(216371776))))[name = string("layers_2_linear_attn_out_proj_weight_cast_fp16")];
+            tensor<fp16, [9216, 2560]> layers_2_mlp_gate_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint4, [9216, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(216376960))), offset = tensor<uint4, [9216, 160]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(56898432))), scale = tensor<fp16, [9216, 160]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(228173504))))[name = string("layers_2_mlp_gate_proj_weight_cast_fp16")];
+            tensor<fp16, [9216, 2560]> layers_2_mlp_up_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint4, [9216, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(231122688))), offset = tensor<uint4, [9216, 160]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(56898432))), scale = tensor<fp16, [9216, 160]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(242919232))))[name = string("layers_2_mlp_up_proj_weight_cast_fp16")];
+            tensor<fp16, [2560, 9216]> layers_2_mlp_down_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint4, [2560, 9216]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(245868416))), offset = tensor<uint4, [2560, 576]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(87127232))), scale = tensor<fp16, [2560, 576]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(257664960))))[name = string("layers_2_mlp_down_proj_weight_cast_fp16")];
+            tensor<fp16, [8192, 2560]> layers_3_self_attn_q_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [8192, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(260614144))), offset = tensor<uint8, [8192, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(20988096))), scale = tensor<fp16, [8192, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(281585728))))[name = string("layers_3_self_attn_q_proj_weight_cast_fp16")];
+            tensor<fp16, [1024, 2560]> layers_3_self_attn_k_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [1024, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(281602176))), offset = tensor<uint8, [1024, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(284225792))), scale = tensor<fp16, [1024, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(284223680))))[name = string("layers_3_self_attn_k_proj_weight_cast_fp16")];
+            tensor<fp16, [1024, 2560]> layers_3_self_attn_v_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [1024, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(284226880))), offset = tensor<uint8, [1024, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(284225792))), scale = tensor<fp16, [1024, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(286848384))))[name = string("layers_3_self_attn_v_proj_weight_cast_fp16")];
+            tensor<fp16, [2560, 4096]> layers_3_self_attn_o_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [2560, 4096]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(286850496))), offset = tensor<uint8, [2560, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(42150080))), scale = tensor<fp16, [2560, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(297336320))))[name = string("layers_3_self_attn_o_proj_weight_cast_fp16")];
+            tensor<fp16, [9216, 2560]> layers_3_mlp_gate_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint4, [9216, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(297341504))), offset = tensor<uint4, [9216, 160]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(56898432))), scale = tensor<fp16, [9216, 160]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(309138048))))[name = string("layers_3_mlp_gate_proj_weight_cast_fp16")];
+            tensor<fp16, [9216, 2560]> layers_3_mlp_up_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint4, [9216, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(312087232))), offset = tensor<uint4, [9216, 160]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(56898432))), scale = tensor<fp16, [9216, 160]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(323883776))))[name = string("layers_3_mlp_up_proj_weight_cast_fp16")];
+            tensor<fp16, [2560, 9216]> layers_3_mlp_down_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint4, [2560, 9216]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(326832960))), offset = tensor<uint4, [2560, 576]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(87127232))), scale = tensor<fp16, [2560, 576]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(338629504))))[name = string("layers_3_mlp_down_proj_weight_cast_fp16")];
+            tensor<fp16, [8192, 2560]> layers_4_linear_attn_in_proj_qkv_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [8192, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(341578688))), offset = tensor<uint8, [8192, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(20988096))), scale = tensor<fp16, [8192, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(362550272))))[name = string("layers_4_linear_attn_in_proj_qkv_weight_cast_fp16")];
+            tensor<fp16, [4096, 2560]> layers_4_linear_attn_in_proj_z_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [4096, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(362566720))), offset = tensor<uint8, [4096, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(31490432))), scale = tensor<fp16, [4096, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(373052544))))[name = string("layers_4_linear_attn_in_proj_z_weight_cast_fp16")];
+            tensor<fp16, [32, 2560]> layers_4_linear_attn_in_proj_b_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [32, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(373060800))), offset = tensor<uint8, [32, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(373142912))), scale = tensor<fp16, [32, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(373142784))))[name = string("layers_4_linear_attn_in_proj_b_weight_cast_fp16")];
+            tensor<fp16, [32, 2560]> layers_4_linear_attn_in_proj_a_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [32, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(373143040))), offset = tensor<uint8, [32, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(373225152))), scale = tensor<fp16, [32, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(373225024))))[name = string("layers_4_linear_attn_in_proj_a_weight_cast_fp16")];
+            tensor<fp16, [2560, 4096]> layers_4_linear_attn_out_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [2560, 4096]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(373225280))), offset = tensor<uint8, [2560, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(42150080))), scale = tensor<fp16, [2560, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(383711104))))[name = string("layers_4_linear_attn_out_proj_weight_cast_fp16")];
+            tensor<fp16, [9216, 2560]> layers_4_mlp_gate_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint4, [9216, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(383716288))), offset = tensor<uint4, [9216, 160]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(56898432))), scale = tensor<fp16, [9216, 160]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(395512832))))[name = string("layers_4_mlp_gate_proj_weight_cast_fp16")];
+            tensor<fp16, [9216, 2560]> layers_4_mlp_up_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint4, [9216, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(398462016))), offset = tensor<uint4, [9216, 160]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(56898432))), scale = tensor<fp16, [9216, 160]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(410258560))))[name = string("layers_4_mlp_up_proj_weight_cast_fp16")];
+            tensor<fp16, [2560, 9216]> layers_4_mlp_down_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint4, [2560, 9216]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(413207744))), offset = tensor<uint4, [2560, 576]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(87127232))), scale = tensor<fp16, [2560, 576]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(425004288))))[name = string("layers_4_mlp_down_proj_weight_cast_fp16")];
+            tensor<fp16, [8192, 2560]> layers_5_linear_attn_in_proj_qkv_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [8192, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(427953472))), offset = tensor<uint8, [8192, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(20988096))), scale = tensor<fp16, [8192, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(448925056))))[name = string("layers_5_linear_attn_in_proj_qkv_weight_cast_fp16")];
+            tensor<fp16, [4096, 2560]> layers_5_linear_attn_in_proj_z_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [4096, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(448941504))), offset = tensor<uint8, [4096, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(31490432))), scale = tensor<fp16, [4096, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(459427328))))[name = string("layers_5_linear_attn_in_proj_z_weight_cast_fp16")];
+            tensor<fp16, [32, 2560]> layers_5_linear_attn_in_proj_b_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [32, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(459435584))), offset = tensor<uint8, [32, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(459517696))), scale = tensor<fp16, [32, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(459517568))))[name = string("layers_5_linear_attn_in_proj_b_weight_cast_fp16")];
+            tensor<fp16, [32, 2560]> layers_5_linear_attn_in_proj_a_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [32, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(459517824))), offset = tensor<uint8, [32, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(459599936))), scale = tensor<fp16, [32, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(459599808))))[name = string("layers_5_linear_attn_in_proj_a_weight_cast_fp16")];
+            tensor<fp16, [2560, 4096]> layers_5_linear_attn_out_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [2560, 4096]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(459600064))), offset = tensor<uint8, [2560, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(42150080))), scale = tensor<fp16, [2560, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(470085888))))[name = string("layers_5_linear_attn_out_proj_weight_cast_fp16")];
+            tensor<fp16, [9216, 2560]> layers_5_mlp_gate_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint4, [9216, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(470091072))), offset = tensor<uint4, [9216, 160]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(56898432))), scale = tensor<fp16, [9216, 160]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(481887616))))[name = string("layers_5_mlp_gate_proj_weight_cast_fp16")];
+            tensor<fp16, [9216, 2560]> layers_5_mlp_up_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint4, [9216, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(484836800))), offset = tensor<uint4, [9216, 160]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(56898432))), scale = tensor<fp16, [9216, 160]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(496633344))))[name = string("layers_5_mlp_up_proj_weight_cast_fp16")];
+            tensor<fp16, [2560, 9216]> layers_5_mlp_down_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint4, [2560, 9216]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(499582528))), offset = tensor<uint4, [2560, 576]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(87127232))), scale = tensor<fp16, [2560, 576]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(511379072))))[name = string("layers_5_mlp_down_proj_weight_cast_fp16")];
+            tensor<fp16, [8192, 2560]> layers_6_linear_attn_in_proj_qkv_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [8192, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(514328256))), offset = tensor<uint8, [8192, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(20988096))), scale = tensor<fp16, [8192, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(535299840))))[name = string("layers_6_linear_attn_in_proj_qkv_weight_cast_fp16")];
+            tensor<fp16, [4096, 2560]> layers_6_linear_attn_in_proj_z_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [4096, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(535316288))), offset = tensor<uint8, [4096, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(31490432))), scale = tensor<fp16, [4096, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(545802112))))[name = string("layers_6_linear_attn_in_proj_z_weight_cast_fp16")];
+            tensor<fp16, [32, 2560]> layers_6_linear_attn_in_proj_b_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [32, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(545810368))), offset = tensor<uint8, [32, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(545892480))), scale = tensor<fp16, [32, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(545892352))))[name = string("layers_6_linear_attn_in_proj_b_weight_cast_fp16")];
+            tensor<fp16, [32, 2560]> layers_6_linear_attn_in_proj_a_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [32, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(545892608))), offset = tensor<uint8, [32, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(545974720))), scale = tensor<fp16, [32, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(545974592))))[name = string("layers_6_linear_attn_in_proj_a_weight_cast_fp16")];
+            tensor<fp16, [2560, 4096]> layers_6_linear_attn_out_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [2560, 4096]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(545974848))), offset = tensor<uint8, [2560, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(42150080))), scale = tensor<fp16, [2560, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(556460672))))[name = string("layers_6_linear_attn_out_proj_weight_cast_fp16")];
+            tensor<fp16, [9216, 2560]> layers_6_mlp_gate_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint4, [9216, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(556465856))), offset = tensor<uint4, [9216, 160]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(56898432))), scale = tensor<fp16, [9216, 160]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(568262400))))[name = string("layers_6_mlp_gate_proj_weight_cast_fp16")];
+            tensor<fp16, [9216, 2560]> layers_6_mlp_up_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint4, [9216, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(571211584))), offset = tensor<uint4, [9216, 160]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(56898432))), scale = tensor<fp16, [9216, 160]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(583008128))))[name = string("layers_6_mlp_up_proj_weight_cast_fp16")];
+            tensor<fp16, [2560, 9216]> layers_6_mlp_down_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint4, [2560, 9216]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(585957312))), offset = tensor<uint4, [2560, 576]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(87127232))), scale = tensor<fp16, [2560, 576]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(597753856))))[name = string("layers_6_mlp_down_proj_weight_cast_fp16")];
+            tensor<fp16, [8192, 2560]> layers_7_self_attn_q_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [8192, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(600703040))), offset = tensor<uint8, [8192, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(20988096))), scale = tensor<fp16, [8192, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(621674624))))[name = string("layers_7_self_attn_q_proj_weight_cast_fp16")];
+            tensor<fp16, [1024, 2560]> layers_7_self_attn_k_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [1024, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(621691072))), offset = tensor<uint8, [1024, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(284225792))), scale = tensor<fp16, [1024, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(624312576))))[name = string("layers_7_self_attn_k_proj_weight_cast_fp16")];
+            tensor<fp16, [1024, 2560]> layers_7_self_attn_v_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [1024, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(624314688))), offset = tensor<uint8, [1024, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(284225792))), scale = tensor<fp16, [1024, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(626936192))))[name = string("layers_7_self_attn_v_proj_weight_cast_fp16")];
+            tensor<fp16, [2560, 4096]> layers_7_self_attn_o_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint8, [2560, 4096]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(626938304))), offset = tensor<uint8, [2560, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(42150080))), scale = tensor<fp16, [2560, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(637424128))))[name = string("layers_7_self_attn_o_proj_weight_cast_fp16")];
+            tensor<fp16, [9216, 2560]> layers_7_mlp_gate_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint4, [9216, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(637429312))), offset = tensor<uint4, [9216, 160]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(56898432))), scale = tensor<fp16, [9216, 160]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(649225856))))[name = string("layers_7_mlp_gate_proj_weight_cast_fp16")];
+            tensor<fp16, [9216, 2560]> layers_7_mlp_up_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint4, [9216, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(652175040))), offset = tensor<uint4, [9216, 160]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(56898432))), scale = tensor<fp16, [9216, 160]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(663971584))))[name = string("layers_7_mlp_up_proj_weight_cast_fp16")];
+            tensor<fp16, [2560, 9216]> layers_7_mlp_down_proj_weight_cast_fp16 = constexpr_blockwise_shift_scale(data = tensor<uint4, [2560, 9216]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(666920768))), offset = tensor<uint4, [2560, 576]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(87127232))), scale = tensor<fp16, [2560, 576]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(678717312))))[name = string("layers_7_mlp_down_proj_weight_cast_fp16")];
+            int32 var_34 = const()[name = string("op_34"), val = int32(-2)];
+            int32 var_49 = const()[name = string("op_49"), val = int32(-1)];
+            fp16 var_48_promoted_to_fp16 = const()[name = string("op_48_promoted_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 1024, 2560]> var_58_cast_fp16 = pow(x = hidden, y = var_48_promoted_to_fp16)[name = string("op_58_cast_fp16")];
+            tensor<int32, [1]> var_60_axes_0 = const()[name = string("op_60_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_60_keep_dims_0 = const()[name = string("op_60_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 1024, 1]> var_60_cast_fp16 = reduce_mean(axes = var_60_axes_0, keep_dims = var_60_keep_dims_0, x = var_58_cast_fp16)[name = string("op_60_cast_fp16")];
+            fp16 var_61_to_fp16 = const()[name = string("op_61_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 1024, 1]> var_62_cast_fp16 = add(x = var_60_cast_fp16, y = var_61_to_fp16)[name = string("op_62_cast_fp16")];
+            fp32 var_63_epsilon_0 = const()[name = string("op_63_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 1024, 1]> var_63_cast_fp16 = rsqrt(epsilon = var_63_epsilon_0, x = var_62_cast_fp16)[name = string("op_63_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> x_1_cast_fp16 = mul(x = hidden, y = var_63_cast_fp16)[name = string("x_1_cast_fp16")];
+            tensor<fp16, [2560]> var_66_to_fp16 = const()[name = string("op_66_to_fp16"), val = tensor<fp16, [2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(681666496)))];
+            tensor<fp16, [1, 1024, 2560]> x_3_cast_fp16 = mul(x = x_1_cast_fp16, y = var_66_to_fp16)[name = string("x_3_cast_fp16")];
+            tensor<fp16, [8192]> linear_0_bias_0_to_fp16 = const()[name = string("linear_0_bias_0_to_fp16"), val = tensor<fp16, [8192]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(681671680)))];
+            tensor<fp16, [1, 1024, 8192]> linear_0_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = layers_0_linear_attn_in_proj_qkv_weight_cast_fp16, x = x_3_cast_fp16)[name = string("linear_0_cast_fp16")];
+            tensor<int32, [3]> input_1_perm_0 = const()[name = string("input_1_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
+            tensor<int32, [6]> input_3_pad_0 = const()[name = string("input_3_pad_0"), val = tensor<int32, [6]>([0, 0, 0, 0, 3, 0])];
+            string input_3_mode_0 = const()[name = string("input_3_mode_0"), val = string("constant")];
+            fp16 const_1_to_fp16 = const()[name = string("const_1_to_fp16"), val = fp16(0x0p+0)];
+            tensor<fp16, [1, 8192, 1024]> input_1_cast_fp16 = transpose(perm = input_1_perm_0, x = linear_0_cast_fp16)[name = string("transpose_55")];
+            tensor<fp16, [1, 8192, 1027]> input_3_cast_fp16 = pad(constant_val = const_1_to_fp16, mode = input_3_mode_0, pad = input_3_pad_0, x = input_1_cast_fp16)[name = string("input_3_cast_fp16")];
+            string input_5_pad_type_0 = const()[name = string("input_5_pad_type_0"), val = string("valid")];
+            int32 input_5_groups_0 = const()[name = string("input_5_groups_0"), val = int32(8192)];
+            tensor<int32, [1]> input_5_strides_0 = const()[name = string("input_5_strides_0"), val = tensor<int32, [1]>([1])];
+            tensor<int32, [2]> input_5_pad_0 = const()[name = string("input_5_pad_0"), val = tensor<int32, [2]>([0, 0])];
+            tensor<int32, [1]> input_5_dilations_0 = const()[name = string("input_5_dilations_0"), val = tensor<int32, [1]>([1])];
+            tensor<fp16, [8192, 1, 4]> layers_0_linear_attn_conv1d_weight_to_fp16 = const()[name = string("layers_0_linear_attn_conv1d_weight_to_fp16"), val = tensor<fp16, [8192, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(681688128)))];
+            tensor<fp16, [1, 8192, 1024]> input_5_cast_fp16 = conv(dilations = input_5_dilations_0, groups = input_5_groups_0, pad = input_5_pad_0, pad_type = input_5_pad_type_0, strides = input_5_strides_0, weight = layers_0_linear_attn_conv1d_weight_to_fp16, x = input_3_cast_fp16)[name = string("input_5_cast_fp16")];
+            tensor<fp16, [1, 8192, 1024]> var_101_cast_fp16 = silu(x = input_5_cast_fp16)[name = string("op_101_cast_fp16")];
+            tensor<int32, [3]> qkv_1_perm_0 = const()[name = string("qkv_1_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
+            tensor<int32, [3]> var_103_begin_0 = const()[name = string("op_103_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_103_end_0 = const()[name = string("op_103_end_0"), val = tensor<int32, [3]>([1, 1024, 2048])];
+            tensor<bool, [3]> var_103_end_mask_0 = const()[name = string("op_103_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
+            tensor<fp16, [1, 1024, 8192]> qkv_1_cast_fp16 = transpose(perm = qkv_1_perm_0, x = var_101_cast_fp16)[name = string("transpose_54")];
+            tensor<fp16, [1, 1024, 2048]> var_103_cast_fp16 = slice_by_index(begin = var_103_begin_0, end = var_103_end_0, end_mask = var_103_end_mask_0, x = qkv_1_cast_fp16)[name = string("op_103_cast_fp16")];
+            tensor<int32, [4]> var_104 = const()[name = string("op_104"), val = tensor<int32, [4]>([1, 1024, 16, 128])];
+            tensor<fp16, [1, 1024, 16, 128]> q_1_cast_fp16 = reshape(shape = var_104, x = var_103_cast_fp16)[name = string("q_1_cast_fp16")];
+            tensor<int32, [3]> var_106_begin_0 = const()[name = string("op_106_begin_0"), val = tensor<int32, [3]>([0, 0, 2048])];
+            tensor<int32, [3]> var_106_end_0 = const()[name = string("op_106_end_0"), val = tensor<int32, [3]>([1, 1024, 4096])];
+            tensor<bool, [3]> var_106_end_mask_0 = const()[name = string("op_106_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
+            tensor<fp16, [1, 1024, 2048]> var_106_cast_fp16 = slice_by_index(begin = var_106_begin_0, end = var_106_end_0, end_mask = var_106_end_mask_0, x = qkv_1_cast_fp16)[name = string("op_106_cast_fp16")];
+            tensor<int32, [4]> var_107 = const()[name = string("op_107"), val = tensor<int32, [4]>([1, 1024, 16, 128])];
+            tensor<fp16, [1, 1024, 16, 128]> k_1_cast_fp16 = reshape(shape = var_107, x = var_106_cast_fp16)[name = string("k_1_cast_fp16")];
+            tensor<int32, [3]> var_109_begin_0 = const()[name = string("op_109_begin_0"), val = tensor<int32, [3]>([0, 0, 4096])];
+            tensor<int32, [3]> var_109_end_0 = const()[name = string("op_109_end_0"), val = tensor<int32, [3]>([1, 1024, 8192])];
+            tensor<bool, [3]> var_109_end_mask_0 = const()[name = string("op_109_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
+            tensor<fp16, [1, 1024, 4096]> var_109_cast_fp16 = slice_by_index(begin = var_109_begin_0, end = var_109_end_0, end_mask = var_109_end_mask_0, x = qkv_1_cast_fp16)[name = string("op_109_cast_fp16")];
+            tensor<int32, [4]> var_110 = const()[name = string("op_110"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<fp16, [1, 1024, 32, 128]> v_1_cast_fp16 = reshape(shape = var_110, x = var_109_cast_fp16)[name = string("v_1_cast_fp16")];
+            tensor<fp16, [4096]> linear_1_bias_0_to_fp16 = const()[name = string("linear_1_bias_0_to_fp16"), val = tensor<fp16, [4096]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(681753728)))];
+            tensor<fp16, [1, 1024, 4096]> linear_1_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = layers_0_linear_attn_in_proj_z_weight_cast_fp16, x = x_3_cast_fp16)[name = string("linear_1_cast_fp16")];
+            tensor<int32, [4]> var_114 = const()[name = string("op_114"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<fp16, [1, 1024, 32, 128]> z_1_cast_fp16 = reshape(shape = var_114, x = linear_1_cast_fp16)[name = string("z_1_cast_fp16")];
+            tensor<fp16, [32]> linear_2_bias_0_to_fp16 = const()[name = string("linear_2_bias_0_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(681761984)))];
+            tensor<fp16, [1, 1024, 32]> linear_2_cast_fp16 = linear(bias = linear_2_bias_0_to_fp16, weight = layers_0_linear_attn_in_proj_b_weight_cast_fp16, x = x_3_cast_fp16)[name = string("linear_2_cast_fp16")];
+            tensor<fp16, [1, 1024, 32]> beta_1_cast_fp16 = sigmoid(x = linear_2_cast_fp16)[name = string("beta_1_cast_fp16")];
+            tensor<fp16, [32]> linear_3_bias_0_to_fp16 = const()[name = string("linear_3_bias_0_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(681762112)))];
+            tensor<fp16, [1, 1024, 32]> linear_3_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = layers_0_linear_attn_in_proj_a_weight_cast_fp16, x = x_3_cast_fp16)[name = string("linear_3_cast_fp16")];
+            tensor<fp16, [32]> layers_0_linear_attn_dt_bias_to_fp16 = const()[name = string("layers_0_linear_attn_dt_bias_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(681762240)))];
+            tensor<fp16, [1, 1024, 32]> var_123_cast_fp16 = add(x = linear_3_cast_fp16, y = layers_0_linear_attn_dt_bias_to_fp16)[name = string("op_123_cast_fp16")];
+            tensor<fp16, [1, 1024, 32]> var_124_cast_fp16 = softplus(x = var_123_cast_fp16)[name = string("op_124_cast_fp16")];
+            tensor<fp16, [32]> var_120_to_fp16 = const()[name = string("op_120_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(681762368)))];
+            tensor<fp16, [1, 1024, 32]> g_1_cast_fp16 = mul(x = var_120_to_fp16, y = var_124_cast_fp16)[name = string("g_1_cast_fp16")];
+            tensor<fp16, [1, 1024, 16, 128]> var_126_cast_fp16 = mul(x = q_1_cast_fp16, y = q_1_cast_fp16)[name = string("op_126_cast_fp16")];
+            tensor<int32, [1]> var_128_axes_0 = const()[name = string("op_128_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_128_keep_dims_0 = const()[name = string("op_128_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 1024, 16, 1]> var_128_cast_fp16 = reduce_sum(axes = var_128_axes_0, keep_dims = var_128_keep_dims_0, x = var_126_cast_fp16)[name = string("op_128_cast_fp16")];
+            fp16 var_129_to_fp16 = const()[name = string("op_129_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 1024, 16, 1]> var_130_cast_fp16 = add(x = var_128_cast_fp16, y = var_129_to_fp16)[name = string("op_130_cast_fp16")];
+            fp32 var_131_epsilon_0 = const()[name = string("op_131_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 1024, 16, 1]> var_131_cast_fp16 = rsqrt(epsilon = var_131_epsilon_0, x = var_130_cast_fp16)[name = string("op_131_cast_fp16")];
+            tensor<fp16, [1, 1024, 16, 128]> q_3_cast_fp16 = mul(x = q_1_cast_fp16, y = var_131_cast_fp16)[name = string("q_3_cast_fp16")];
+            tensor<fp16, [1, 1024, 16, 128]> var_133_cast_fp16 = mul(x = k_1_cast_fp16, y = k_1_cast_fp16)[name = string("op_133_cast_fp16")];
+            tensor<int32, [1]> var_135_axes_0 = const()[name = string("op_135_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_135_keep_dims_0 = const()[name = string("op_135_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 1024, 16, 1]> var_135_cast_fp16 = reduce_sum(axes = var_135_axes_0, keep_dims = var_135_keep_dims_0, x = var_133_cast_fp16)[name = string("op_135_cast_fp16")];
+            fp16 var_136_to_fp16 = const()[name = string("op_136_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 1024, 16, 1]> var_137_cast_fp16 = add(x = var_135_cast_fp16, y = var_136_to_fp16)[name = string("op_137_cast_fp16")];
+            fp32 var_138_epsilon_0 = const()[name = string("op_138_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 1024, 16, 1]> var_138_cast_fp16 = rsqrt(epsilon = var_138_epsilon_0, x = var_137_cast_fp16)[name = string("op_138_cast_fp16")];
+            tensor<fp16, [1, 1024, 16, 128]> k_3_cast_fp16 = mul(x = k_1_cast_fp16, y = var_138_cast_fp16)[name = string("k_3_cast_fp16")];
+            tensor<int32, [1]> var_143_axes_0 = const()[name = string("op_143_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [1, 1024, 16, 1, 128]> var_143_cast_fp16 = expand_dims(axes = var_143_axes_0, x = q_3_cast_fp16)[name = string("op_143_cast_fp16")];
+            tensor<int32, [5]> var_145_reps_0 = const()[name = string("op_145_reps_0"), val = tensor<int32, [5]>([1, 1, 1, 2, 1])];
+            tensor<fp16, [1, 1024, 16, 2, 128]> var_145_cast_fp16 = tile(reps = var_145_reps_0, x = var_143_cast_fp16)[name = string("op_145_cast_fp16")];
+            tensor<int32, [4]> var_146 = const()[name = string("op_146"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<fp16, [1, 1024, 32, 128]> q_5_cast_fp16 = reshape(shape = var_146, x = var_145_cast_fp16)[name = string("q_5_cast_fp16")];
+            tensor<int32, [1]> var_151_axes_0 = const()[name = string("op_151_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [1, 1024, 16, 1, 128]> var_151_cast_fp16 = expand_dims(axes = var_151_axes_0, x = k_3_cast_fp16)[name = string("op_151_cast_fp16")];
+            tensor<int32, [5]> var_153_reps_0 = const()[name = string("op_153_reps_0"), val = tensor<int32, [5]>([1, 1, 1, 2, 1])];
+            tensor<fp16, [1, 1024, 16, 2, 128]> var_153_cast_fp16 = tile(reps = var_153_reps_0, x = var_151_cast_fp16)[name = string("op_153_cast_fp16")];
+            tensor<int32, [4]> var_154 = const()[name = string("op_154"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<fp16, [1, 1024, 32, 128]> k_5_cast_fp16 = reshape(shape = var_154, x = var_153_cast_fp16)[name = string("k_5_cast_fp16")];
+            fp16 var_156_to_fp16 = const()[name = string("op_156_to_fp16"), val = fp16(0x1.6ap-4)];
+            tensor<fp16, [1, 1024, 32, 128]> q_7_cast_fp16 = mul(x = q_5_cast_fp16, y = var_156_to_fp16)[name = string("q_7_cast_fp16")];
+            tensor<int32, [4]> var_158_begin_0 = const()[name = string("op_158_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_158_end_0 = const()[name = string("op_158_end_0"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<bool, [4]> var_158_end_mask_0 = const()[name = string("op_158_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_158_squeeze_mask_0 = const()[name = string("op_158_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [1024, 32, 128]> var_158_cast_fp16 = slice_by_index(begin = var_158_begin_0, end = var_158_end_0, end_mask = var_158_end_mask_0, squeeze_mask = var_158_squeeze_mask_0, x = q_7_cast_fp16)[name = string("op_158_cast_fp16")];
+            tensor<int32, [3]> var_159 = const()[name = string("op_159"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_161 = const()[name = string("op_161"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<fp16, [32, 1024, 128]> var_160_cast_fp16 = transpose(perm = var_159, x = var_158_cast_fp16)[name = string("transpose_53")];
+            tensor<fp16, [32, 16, 64, 128]> q_9_cast_fp16 = reshape(shape = var_161, x = var_160_cast_fp16)[name = string("q_9_cast_fp16")];
+            tensor<int32, [4]> var_163_begin_0 = const()[name = string("op_163_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_163_end_0 = const()[name = string("op_163_end_0"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<bool, [4]> var_163_end_mask_0 = const()[name = string("op_163_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_163_squeeze_mask_0 = const()[name = string("op_163_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [1024, 32, 128]> var_163_cast_fp16 = slice_by_index(begin = var_163_begin_0, end = var_163_end_0, end_mask = var_163_end_mask_0, squeeze_mask = var_163_squeeze_mask_0, x = k_5_cast_fp16)[name = string("op_163_cast_fp16")];
+            tensor<int32, [3]> var_164 = const()[name = string("op_164"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_166 = const()[name = string("op_166"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<fp16, [32, 1024, 128]> var_165_cast_fp16 = transpose(perm = var_164, x = var_163_cast_fp16)[name = string("transpose_52")];
+            tensor<fp16, [32, 16, 64, 128]> k_7_cast_fp16 = reshape(shape = var_166, x = var_165_cast_fp16)[name = string("k_7_cast_fp16")];
+            tensor<int32, [4]> var_168_begin_0 = const()[name = string("op_168_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_168_end_0 = const()[name = string("op_168_end_0"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<bool, [4]> var_168_end_mask_0 = const()[name = string("op_168_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_168_squeeze_mask_0 = const()[name = string("op_168_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [1024, 32, 128]> var_168_cast_fp16 = slice_by_index(begin = var_168_begin_0, end = var_168_end_0, end_mask = var_168_end_mask_0, squeeze_mask = var_168_squeeze_mask_0, x = v_1_cast_fp16)[name = string("op_168_cast_fp16")];
+            tensor<int32, [3]> var_169 = const()[name = string("op_169"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_171 = const()[name = string("op_171"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<fp16, [32, 1024, 128]> var_170_cast_fp16 = transpose(perm = var_169, x = var_168_cast_fp16)[name = string("transpose_51")];
+            tensor<fp16, [32, 16, 64, 128]> v_3_cast_fp16 = reshape(shape = var_171, x = var_170_cast_fp16)[name = string("v_3_cast_fp16")];
+            tensor<int32, [3]> var_173_begin_0 = const()[name = string("op_173_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_173_end_0 = const()[name = string("op_173_end_0"), val = tensor<int32, [3]>([1, 1024, 32])];
+            tensor<bool, [3]> var_173_end_mask_0 = const()[name = string("op_173_end_mask_0"), val = tensor<bool, [3]>([false, true, true])];
+            tensor<bool, [3]> var_173_squeeze_mask_0 = const()[name = string("op_173_squeeze_mask_0"), val = tensor<bool, [3]>([true, false, false])];
+            tensor<fp16, [1024, 32]> var_173_cast_fp16 = slice_by_index(begin = var_173_begin_0, end = var_173_end_0, end_mask = var_173_end_mask_0, squeeze_mask = var_173_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_173_cast_fp16")];
+            tensor<int32, [2]> var_174_perm_0 = const()[name = string("op_174_perm_0"), val = tensor<int32, [2]>([1, 0])];
+            tensor<int32, [4]> var_175 = const()[name = string("op_175"), val = tensor<int32, [4]>([32, 16, 64, 1])];
+            tensor<fp16, [32, 1024]> var_174_cast_fp16 = transpose(perm = var_174_perm_0, x = var_173_cast_fp16)[name = string("transpose_50")];
+            tensor<fp16, [32, 16, 64, 1]> beta_3_cast_fp16 = reshape(shape = var_175, x = var_174_cast_fp16)[name = string("beta_3_cast_fp16")];
+            tensor<int32, [3]> var_177_begin_0 = const()[name = string("op_177_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_177_end_0 = const()[name = string("op_177_end_0"), val = tensor<int32, [3]>([1, 1024, 32])];
+            tensor<bool, [3]> var_177_end_mask_0 = const()[name = string("op_177_end_mask_0"), val = tensor<bool, [3]>([false, true, true])];
+            tensor<bool, [3]> var_177_squeeze_mask_0 = const()[name = string("op_177_squeeze_mask_0"), val = tensor<bool, [3]>([true, false, false])];
+            tensor<fp16, [1024, 32]> var_177_cast_fp16 = slice_by_index(begin = var_177_begin_0, end = var_177_end_0, end_mask = var_177_end_mask_0, squeeze_mask = var_177_squeeze_mask_0, x = g_1_cast_fp16)[name = string("op_177_cast_fp16")];
+            tensor<int32, [2]> var_178_perm_0 = const()[name = string("op_178_perm_0"), val = tensor<int32, [2]>([1, 0])];
+            tensor<int32, [3]> var_179 = const()[name = string("op_179"), val = tensor<int32, [3]>([32, 16, 64])];
+            tensor<fp16, [32, 1024]> var_178_cast_fp16 = transpose(perm = var_178_perm_0, x = var_177_cast_fp16)[name = string("transpose_49")];
+            tensor<fp16, [32, 16, 64]> g_3_cast_fp16 = reshape(shape = var_179, x = var_178_cast_fp16)[name = string("g_3_cast_fp16")];
+            tensor<int32, [1]> g_col_1_axes_0 = const()[name = string("g_col_1_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 16, 64, 1]> g_col_1_cast_fp16 = expand_dims(axes = g_col_1_axes_0, x = g_3_cast_fp16)[name = string("g_col_1_cast_fp16")];
+            bool var_182_transpose_x_0 = const()[name = string("op_182_transpose_x_0"), val = bool(false)];
+            bool var_182_transpose_y_0 = const()[name = string("op_182_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [64, 64]> layers_0_linear_attn_lower_incl_to_fp16 = const()[name = string("layers_0_linear_attn_lower_incl_to_fp16"), val = tensor<fp16, [64, 64]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(681762496)))];
+            tensor<fp16, [32, 16, 64, 1]> var_182_cast_fp16 = matmul(transpose_x = var_182_transpose_x_0, transpose_y = var_182_transpose_y_0, x = layers_0_linear_attn_lower_incl_to_fp16, y = g_col_1_cast_fp16)[name = string("op_182_cast_fp16")];
+            tensor<int32, [1]> cum_1_axes_0 = const()[name = string("cum_1_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 16, 64]> cum_1_cast_fp16 = squeeze(axes = cum_1_axes_0, x = var_182_cast_fp16)[name = string("cum_1_cast_fp16")];
+            bool var_184_transpose_x_0 = const()[name = string("op_184_transpose_x_0"), val = bool(false)];
+            bool var_184_transpose_y_0 = const()[name = string("op_184_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [64, 64]> layers_0_linear_attn_suffix_sum_t_to_fp16 = const()[name = string("layers_0_linear_attn_suffix_sum_t_to_fp16"), val = tensor<fp16, [64, 64]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(681770752)))];
+            tensor<fp16, [32, 16, 64, 1]> var_184_cast_fp16 = matmul(transpose_x = var_184_transpose_x_0, transpose_y = var_184_transpose_y_0, x = layers_0_linear_attn_suffix_sum_t_to_fp16, y = g_col_1_cast_fp16)[name = string("op_184_cast_fp16")];
+            tensor<int32, [1]> to_end_1_axes_0 = const()[name = string("to_end_1_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 16, 64]> to_end_1_cast_fp16 = squeeze(axes = to_end_1_axes_0, x = var_184_cast_fp16)[name = string("to_end_1_cast_fp16")];
+            bool var_186_transpose_x_0 = const()[name = string("op_186_transpose_x_0"), val = bool(false)];
+            bool var_186_transpose_y_0 = const()[name = string("op_186_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [4096, 64]> layers_0_linear_attn_pair_sum_t_to_fp16 = const()[name = string("layers_0_linear_attn_pair_sum_t_to_fp16"), val = tensor<fp16, [4096, 64]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(681779008)))];
+            tensor<fp16, [32, 16, 4096, 1]> var_186_cast_fp16 = matmul(transpose_x = var_186_transpose_x_0, transpose_y = var_186_transpose_y_0, x = layers_0_linear_attn_pair_sum_t_to_fp16, y = g_col_1_cast_fp16)[name = string("op_186_cast_fp16")];
+            tensor<int32, [4]> var_187 = const()[name = string("op_187"), val = tensor<int32, [4]>([32, 16, 64, 64])];
+            tensor<fp16, [32, 16, 64, 64]> pair_1_cast_fp16 = reshape(shape = var_187, x = var_186_cast_fp16)[name = string("pair_1_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 64]> var_189_cast_fp16 = exp(x = pair_1_cast_fp16)[name = string("op_189_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 64]> pair_decay_1_cast_fp16 = mul(x = var_189_cast_fp16, y = layers_0_linear_attn_lower_incl_to_fp16)[name = string("pair_decay_1_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 128]> k_beta_1_cast_fp16 = mul(x = k_7_cast_fp16, y = beta_3_cast_fp16)[name = string("k_beta_1_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 128]> v_beta_1_cast_fp16 = mul(x = v_3_cast_fp16, y = beta_3_cast_fp16)[name = string("v_beta_1_cast_fp16")];
+            bool var_194_transpose_x_1 = const()[name = string("op_194_transpose_x_1"), val = bool(false)];
+            bool var_194_transpose_y_1 = const()[name = string("op_194_transpose_y_1"), val = bool(true)];
+            tensor<fp16, [32, 16, 64, 64]> var_194_cast_fp16 = matmul(transpose_x = var_194_transpose_x_1, transpose_y = var_194_transpose_y_1, x = k_beta_1_cast_fp16, y = k_7_cast_fp16)[name = string("op_194_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 64]> kkt_1_cast_fp16 = mul(x = var_194_cast_fp16, y = pair_decay_1_cast_fp16)[name = string("kkt_1_cast_fp16")];
+            bool var_197_transpose_x_1 = const()[name = string("op_197_transpose_x_1"), val = bool(false)];
+            bool var_197_transpose_y_1 = const()[name = string("op_197_transpose_y_1"), val = bool(true)];
+            tensor<fp16, [32, 16, 64, 64]> var_197_cast_fp16 = matmul(transpose_x = var_197_transpose_x_1, transpose_y = var_197_transpose_y_1, x = q_9_cast_fp16, y = k_7_cast_fp16)[name = string("op_197_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 64]> attn_intra_1_cast_fp16 = mul(x = var_197_cast_fp16, y = pair_decay_1_cast_fp16)[name = string("attn_intra_1_cast_fp16")];
+            tensor<fp16, [64, 64]> layers_0_linear_attn_strict_lower_to_fp16 = const()[name = string("layers_0_linear_attn_strict_lower_to_fp16"), val = tensor<fp16, [64, 64]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(682303360)))];
+            tensor<fp16, [32, 16, 64, 64]> var_199_cast_fp16 = mul(x = kkt_1_cast_fp16, y = layers_0_linear_attn_strict_lower_to_fp16)[name = string("op_199_cast_fp16")];
+            tensor<fp16, [64, 64]> layers_0_linear_attn_eye_to_fp16 = const()[name = string("layers_0_linear_attn_eye_to_fp16"), val = tensor<fp16, [64, 64]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(682311616)))];
+            tensor<fp16, [32, 16, 64, 64]> t_1_cast_fp16 = add(x = layers_0_linear_attn_eye_to_fp16, y = var_199_cast_fp16)[name = string("t_1_cast_fp16")];
+            tensor<int32, [3]> var_203 = const()[name = string("op_203"), val = tensor<int32, [3]>([-1, 64, 64])];
+            tensor<fp16, [512, 64, 64]> t_3_cast_fp16 = reshape(shape = var_203, x = t_1_cast_fp16)[name = string("t_3_cast_fp16")];
+            tensor<int32, [5]> var_208 = const()[name = string("op_208"), val = tensor<int32, [5]>([512, 32, 2, 32, 2])];
+            tensor<fp16, [512, 32, 2, 32, 2]> var_209_cast_fp16 = reshape(shape = var_208, x = t_3_cast_fp16)[name = string("op_209_cast_fp16")];
+            tensor<fp16, [32, 1, 32, 1]> var_213_to_fp16 = const()[name = string("op_213_to_fp16"), val = tensor<fp16, [32, 1, 32, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(682319872)))];
+            tensor<fp16, [512, 32, 2, 32, 2]> var_214_cast_fp16 = mul(x = var_209_cast_fp16, y = var_213_to_fp16)[name = string("op_214_cast_fp16")];
+            tensor<int32, [1]> blocks_1_axes_0 = const()[name = string("blocks_1_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_1_keep_dims_0 = const()[name = string("blocks_1_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 32, 2, 2]> blocks_1_cast_fp16 = reduce_sum(axes = blocks_1_axes_0, keep_dims = blocks_1_keep_dims_0, x = var_214_cast_fp16)[name = string("blocks_1_cast_fp16")];
+            tensor<int32, [4]> var_217_begin_0 = const()[name = string("op_217_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
+            tensor<int32, [4]> var_217_end_0 = const()[name = string("op_217_end_0"), val = tensor<int32, [4]>([512, 32, 2, 2])];
+            tensor<bool, [4]> var_217_end_mask_0 = const()[name = string("op_217_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 32, 1, 2]> var_217_cast_fp16 = slice_by_index(begin = var_217_begin_0, end = var_217_end_0, end_mask = var_217_end_mask_0, x = blocks_1_cast_fp16)[name = string("op_217_cast_fp16")];
+            tensor<int32, [4]> x_5_begin_0 = const()[name = string("x_5_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_5_end_0 = const()[name = string("x_5_end_0"), val = tensor<int32, [4]>([512, 32, 1, 1])];
+            tensor<bool, [4]> x_5_end_mask_0 = const()[name = string("x_5_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 32, 1, 1]> x_5_cast_fp16 = slice_by_index(begin = x_5_begin_0, end = x_5_end_0, end_mask = x_5_end_mask_0, x = var_217_cast_fp16)[name = string("x_5_cast_fp16")];
+            bool var_227_transpose_x_0 = const()[name = string("op_227_transpose_x_0"), val = bool(false)];
+            bool var_227_transpose_y_0 = const()[name = string("op_227_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 32, 1, 1]> var_221_to_fp16 = const()[name = string("op_221_to_fp16"), val = tensor<fp16, [512, 32, 1, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(682321984)))];
+            tensor<fp16, [512, 32, 1, 1]> var_227_cast_fp16 = matmul(transpose_x = var_227_transpose_x_0, transpose_y = var_227_transpose_y_0, x = x_5_cast_fp16, y = var_221_to_fp16)[name = string("op_227_cast_fp16")];
+            bool var_228_transpose_x_0 = const()[name = string("op_228_transpose_x_0"), val = bool(false)];
+            bool var_228_transpose_y_0 = const()[name = string("op_228_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 32, 1, 1]> var_228_cast_fp16 = matmul(transpose_x = var_228_transpose_x_0, transpose_y = var_228_transpose_y_0, x = var_221_to_fp16, y = var_227_cast_fp16)[name = string("op_228_cast_fp16")];
+            fp16 const_6_promoted_to_fp16 = const()[name = string("const_6_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 32, 1, 1]> lower_1_cast_fp16 = mul(x = var_228_cast_fp16, y = const_6_promoted_to_fp16)[name = string("lower_1_cast_fp16")];
+            bool bottom_1_interleave_0 = const()[name = string("bottom_1_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 32, 1, 2]> bottom_1_cast_fp16 = concat(axis = var_49, interleave = bottom_1_interleave_0, values = (lower_1_cast_fp16, var_221_to_fp16))[name = string("bottom_1_cast_fp16")];
+            bool inv_3_interleave_0 = const()[name = string("inv_3_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 32, 1, 2]> top_1_to_fp16 = const()[name = string("top_1_to_fp16"), val = tensor<fp16, [512, 32, 1, 2]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(682354816)))];
+            tensor<fp16, [512, 32, 2, 2]> inv_3_cast_fp16 = concat(axis = var_34, interleave = inv_3_interleave_0, values = (top_1_to_fp16, bottom_1_cast_fp16))[name = string("inv_3_cast_fp16")];
+            tensor<int32, [5]> var_237 = const()[name = string("op_237"), val = tensor<int32, [5]>([512, 16, 4, 16, 4])];
+            tensor<fp16, [512, 16, 4, 16, 4]> var_238_cast_fp16 = reshape(shape = var_237, x = t_3_cast_fp16)[name = string("op_238_cast_fp16")];
+            tensor<fp16, [16, 1, 16, 1]> var_242_to_fp16 = const()[name = string("op_242_to_fp16"), val = tensor<fp16, [16, 1, 16, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(682420416)))];
+            tensor<fp16, [512, 16, 4, 16, 4]> var_243_cast_fp16 = mul(x = var_238_cast_fp16, y = var_242_to_fp16)[name = string("op_243_cast_fp16")];
+            tensor<int32, [1]> blocks_3_axes_0 = const()[name = string("blocks_3_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_3_keep_dims_0 = const()[name = string("blocks_3_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 4, 4]> blocks_3_cast_fp16 = reduce_sum(axes = blocks_3_axes_0, keep_dims = blocks_3_keep_dims_0, x = var_243_cast_fp16)[name = string("blocks_3_cast_fp16")];
+            tensor<int32, [4]> var_246_begin_0 = const()[name = string("op_246_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
+            tensor<int32, [4]> var_246_end_0 = const()[name = string("op_246_end_0"), val = tensor<int32, [4]>([512, 16, 4, 4])];
+            tensor<bool, [4]> var_246_end_mask_0 = const()[name = string("op_246_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 16, 2, 4]> var_246_cast_fp16 = slice_by_index(begin = var_246_begin_0, end = var_246_end_0, end_mask = var_246_end_mask_0, x = blocks_3_cast_fp16)[name = string("op_246_cast_fp16")];
+            tensor<int32, [4]> x_7_begin_0 = const()[name = string("x_7_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_7_end_0 = const()[name = string("x_7_end_0"), val = tensor<int32, [4]>([512, 16, 2, 2])];
+            tensor<bool, [4]> x_7_end_mask_0 = const()[name = string("x_7_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 16, 2, 2]> x_7_cast_fp16 = slice_by_index(begin = x_7_begin_0, end = x_7_end_0, end_mask = x_7_end_mask_0, x = var_246_cast_fp16)[name = string("x_7_cast_fp16")];
+            tensor<int32, [5]> var_248 = const()[name = string("op_248"), val = tensor<int32, [5]>([512, 16, 2, 2, 2])];
+            tensor<fp16, [512, 16, 2, 2, 2]> pair_5_cast_fp16 = reshape(shape = var_248, x = inv_3_cast_fp16)[name = string("pair_5_cast_fp16")];
+            tensor<int32, [5]> var_250_begin_0 = const()[name = string("op_250_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_250_end_0 = const()[name = string("op_250_end_0"), val = tensor<int32, [5]>([512, 16, 1, 2, 2])];
+            tensor<bool, [5]> var_250_end_mask_0 = const()[name = string("op_250_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_250_squeeze_mask_0 = const()[name = string("op_250_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 16, 2, 2]> var_250_cast_fp16 = slice_by_index(begin = var_250_begin_0, end = var_250_end_0, end_mask = var_250_end_mask_0, squeeze_mask = var_250_squeeze_mask_0, x = pair_5_cast_fp16)[name = string("op_250_cast_fp16")];
+            tensor<int32, [5]> var_253_begin_0 = const()[name = string("op_253_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_253_end_0 = const()[name = string("op_253_end_0"), val = tensor<int32, [5]>([512, 16, 2, 2, 2])];
+            tensor<bool, [5]> var_253_end_mask_0 = const()[name = string("op_253_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_253_squeeze_mask_0 = const()[name = string("op_253_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 16, 2, 2]> var_253_cast_fp16 = slice_by_index(begin = var_253_begin_0, end = var_253_end_0, end_mask = var_253_end_mask_0, squeeze_mask = var_253_squeeze_mask_0, x = pair_5_cast_fp16)[name = string("op_253_cast_fp16")];
+            bool var_256_transpose_x_0 = const()[name = string("op_256_transpose_x_0"), val = bool(false)];
+            bool var_256_transpose_y_0 = const()[name = string("op_256_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 2, 2]> var_256_cast_fp16 = matmul(transpose_x = var_256_transpose_x_0, transpose_y = var_256_transpose_y_0, x = x_7_cast_fp16, y = var_250_cast_fp16)[name = string("op_256_cast_fp16")];
+            bool var_257_transpose_x_0 = const()[name = string("op_257_transpose_x_0"), val = bool(false)];
+            bool var_257_transpose_y_0 = const()[name = string("op_257_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 2, 2]> var_257_cast_fp16 = matmul(transpose_x = var_257_transpose_x_0, transpose_y = var_257_transpose_y_0, x = var_253_cast_fp16, y = var_256_cast_fp16)[name = string("op_257_cast_fp16")];
+            fp16 const_7_promoted_to_fp16 = const()[name = string("const_7_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 16, 2, 2]> lower_3_cast_fp16 = mul(x = var_257_cast_fp16, y = const_7_promoted_to_fp16)[name = string("lower_3_cast_fp16")];
+            bool top_3_interleave_0 = const()[name = string("top_3_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 2, 2]> var_259_to_fp16 = const()[name = string("op_259_to_fp16"), val = tensor<fp16, [512, 16, 2, 2]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(682420992)))];
+            tensor<fp16, [512, 16, 2, 4]> top_3_cast_fp16 = concat(axis = var_49, interleave = top_3_interleave_0, values = (var_250_cast_fp16, var_259_to_fp16))[name = string("top_3_cast_fp16")];
+            bool bottom_3_interleave_0 = const()[name = string("bottom_3_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 2, 4]> bottom_3_cast_fp16 = concat(axis = var_49, interleave = bottom_3_interleave_0, values = (lower_3_cast_fp16, var_253_cast_fp16))[name = string("bottom_3_cast_fp16")];
+            bool inv_5_interleave_0 = const()[name = string("inv_5_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 4, 4]> inv_5_cast_fp16 = concat(axis = var_34, interleave = inv_5_interleave_0, values = (top_3_cast_fp16, bottom_3_cast_fp16))[name = string("inv_5_cast_fp16")];
+            tensor<int32, [5]> var_266 = const()[name = string("op_266"), val = tensor<int32, [5]>([512, 8, 8, 8, 8])];
+            tensor<fp16, [512, 8, 8, 8, 8]> var_267_cast_fp16 = reshape(shape = var_266, x = t_3_cast_fp16)[name = string("op_267_cast_fp16")];
+            tensor<fp16, [8, 1, 8, 1]> var_271_to_fp16 = const()[name = string("op_271_to_fp16"), val = tensor<fp16, [8, 1, 8, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(682486592)))];
+            tensor<fp16, [512, 8, 8, 8, 8]> var_272_cast_fp16 = mul(x = var_267_cast_fp16, y = var_271_to_fp16)[name = string("op_272_cast_fp16")];
+            tensor<int32, [1]> blocks_5_axes_0 = const()[name = string("blocks_5_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_5_keep_dims_0 = const()[name = string("blocks_5_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 8, 8]> blocks_5_cast_fp16 = reduce_sum(axes = blocks_5_axes_0, keep_dims = blocks_5_keep_dims_0, x = var_272_cast_fp16)[name = string("blocks_5_cast_fp16")];
+            tensor<int32, [4]> var_275_begin_0 = const()[name = string("op_275_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
+            tensor<int32, [4]> var_275_end_0 = const()[name = string("op_275_end_0"), val = tensor<int32, [4]>([512, 8, 8, 8])];
+            tensor<bool, [4]> var_275_end_mask_0 = const()[name = string("op_275_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 8, 4, 8]> var_275_cast_fp16 = slice_by_index(begin = var_275_begin_0, end = var_275_end_0, end_mask = var_275_end_mask_0, x = blocks_5_cast_fp16)[name = string("op_275_cast_fp16")];
+            tensor<int32, [4]> x_9_begin_0 = const()[name = string("x_9_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_9_end_0 = const()[name = string("x_9_end_0"), val = tensor<int32, [4]>([512, 8, 4, 4])];
+            tensor<bool, [4]> x_9_end_mask_0 = const()[name = string("x_9_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 8, 4, 4]> x_9_cast_fp16 = slice_by_index(begin = x_9_begin_0, end = x_9_end_0, end_mask = x_9_end_mask_0, x = var_275_cast_fp16)[name = string("x_9_cast_fp16")];
+            tensor<int32, [5]> var_277 = const()[name = string("op_277"), val = tensor<int32, [5]>([512, 8, 2, 4, 4])];
+            tensor<fp16, [512, 8, 2, 4, 4]> pair_7_cast_fp16 = reshape(shape = var_277, x = inv_5_cast_fp16)[name = string("pair_7_cast_fp16")];
+            tensor<int32, [5]> var_279_begin_0 = const()[name = string("op_279_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_279_end_0 = const()[name = string("op_279_end_0"), val = tensor<int32, [5]>([512, 8, 1, 4, 4])];
+            tensor<bool, [5]> var_279_end_mask_0 = const()[name = string("op_279_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_279_squeeze_mask_0 = const()[name = string("op_279_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 8, 4, 4]> var_279_cast_fp16 = slice_by_index(begin = var_279_begin_0, end = var_279_end_0, end_mask = var_279_end_mask_0, squeeze_mask = var_279_squeeze_mask_0, x = pair_7_cast_fp16)[name = string("op_279_cast_fp16")];
+            tensor<int32, [5]> var_282_begin_0 = const()[name = string("op_282_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_282_end_0 = const()[name = string("op_282_end_0"), val = tensor<int32, [5]>([512, 8, 2, 4, 4])];
+            tensor<bool, [5]> var_282_end_mask_0 = const()[name = string("op_282_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_282_squeeze_mask_0 = const()[name = string("op_282_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 8, 4, 4]> var_282_cast_fp16 = slice_by_index(begin = var_282_begin_0, end = var_282_end_0, end_mask = var_282_end_mask_0, squeeze_mask = var_282_squeeze_mask_0, x = pair_7_cast_fp16)[name = string("op_282_cast_fp16")];
+            bool var_285_transpose_x_0 = const()[name = string("op_285_transpose_x_0"), val = bool(false)];
+            bool var_285_transpose_y_0 = const()[name = string("op_285_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 4, 4]> var_285_cast_fp16 = matmul(transpose_x = var_285_transpose_x_0, transpose_y = var_285_transpose_y_0, x = x_9_cast_fp16, y = var_279_cast_fp16)[name = string("op_285_cast_fp16")];
+            bool var_286_transpose_x_0 = const()[name = string("op_286_transpose_x_0"), val = bool(false)];
+            bool var_286_transpose_y_0 = const()[name = string("op_286_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 4, 4]> var_286_cast_fp16 = matmul(transpose_x = var_286_transpose_x_0, transpose_y = var_286_transpose_y_0, x = var_282_cast_fp16, y = var_285_cast_fp16)[name = string("op_286_cast_fp16")];
+            fp16 const_8_promoted_to_fp16 = const()[name = string("const_8_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 8, 4, 4]> lower_5_cast_fp16 = mul(x = var_286_cast_fp16, y = const_8_promoted_to_fp16)[name = string("lower_5_cast_fp16")];
+            bool top_5_interleave_0 = const()[name = string("top_5_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 4, 4]> var_288_to_fp16 = const()[name = string("op_288_to_fp16"), val = tensor<fp16, [512, 8, 4, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(682486784)))];
+            tensor<fp16, [512, 8, 4, 8]> top_5_cast_fp16 = concat(axis = var_49, interleave = top_5_interleave_0, values = (var_279_cast_fp16, var_288_to_fp16))[name = string("top_5_cast_fp16")];
+            bool bottom_5_interleave_0 = const()[name = string("bottom_5_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 4, 8]> bottom_5_cast_fp16 = concat(axis = var_49, interleave = bottom_5_interleave_0, values = (lower_5_cast_fp16, var_282_cast_fp16))[name = string("bottom_5_cast_fp16")];
+            bool inv_7_interleave_0 = const()[name = string("inv_7_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 8, 8]> inv_7_cast_fp16 = concat(axis = var_34, interleave = inv_7_interleave_0, values = (top_5_cast_fp16, bottom_5_cast_fp16))[name = string("inv_7_cast_fp16")];
+            tensor<int32, [5]> var_295 = const()[name = string("op_295"), val = tensor<int32, [5]>([512, 4, 16, 4, 16])];
+            tensor<fp16, [512, 4, 16, 4, 16]> var_296_cast_fp16 = reshape(shape = var_295, x = t_3_cast_fp16)[name = string("op_296_cast_fp16")];
+            tensor<fp16, [4, 1, 4, 1]> var_300_to_fp16 = const()[name = string("op_300_to_fp16"), val = tensor<fp16, [4, 1, 4, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(682617920)))];
+            tensor<fp16, [512, 4, 16, 4, 16]> var_301_cast_fp16 = mul(x = var_296_cast_fp16, y = var_300_to_fp16)[name = string("op_301_cast_fp16")];
+            tensor<int32, [1]> blocks_7_axes_0 = const()[name = string("blocks_7_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_7_keep_dims_0 = const()[name = string("blocks_7_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 16, 16]> blocks_7_cast_fp16 = reduce_sum(axes = blocks_7_axes_0, keep_dims = blocks_7_keep_dims_0, x = var_301_cast_fp16)[name = string("blocks_7_cast_fp16")];
+            tensor<int32, [4]> var_304_begin_0 = const()[name = string("op_304_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
+            tensor<int32, [4]> var_304_end_0 = const()[name = string("op_304_end_0"), val = tensor<int32, [4]>([512, 4, 16, 16])];
+            tensor<bool, [4]> var_304_end_mask_0 = const()[name = string("op_304_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 4, 8, 16]> var_304_cast_fp16 = slice_by_index(begin = var_304_begin_0, end = var_304_end_0, end_mask = var_304_end_mask_0, x = blocks_7_cast_fp16)[name = string("op_304_cast_fp16")];
+            tensor<int32, [4]> x_11_begin_0 = const()[name = string("x_11_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_11_end_0 = const()[name = string("x_11_end_0"), val = tensor<int32, [4]>([512, 4, 8, 8])];
+            tensor<bool, [4]> x_11_end_mask_0 = const()[name = string("x_11_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 4, 8, 8]> x_11_cast_fp16 = slice_by_index(begin = x_11_begin_0, end = x_11_end_0, end_mask = x_11_end_mask_0, x = var_304_cast_fp16)[name = string("x_11_cast_fp16")];
+            tensor<int32, [5]> var_306 = const()[name = string("op_306"), val = tensor<int32, [5]>([512, 4, 2, 8, 8])];
+            tensor<fp16, [512, 4, 2, 8, 8]> pair_9_cast_fp16 = reshape(shape = var_306, x = inv_7_cast_fp16)[name = string("pair_9_cast_fp16")];
+            tensor<int32, [5]> var_308_begin_0 = const()[name = string("op_308_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_308_end_0 = const()[name = string("op_308_end_0"), val = tensor<int32, [5]>([512, 4, 1, 8, 8])];
+            tensor<bool, [5]> var_308_end_mask_0 = const()[name = string("op_308_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_308_squeeze_mask_0 = const()[name = string("op_308_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 4, 8, 8]> var_308_cast_fp16 = slice_by_index(begin = var_308_begin_0, end = var_308_end_0, end_mask = var_308_end_mask_0, squeeze_mask = var_308_squeeze_mask_0, x = pair_9_cast_fp16)[name = string("op_308_cast_fp16")];
+            tensor<int32, [5]> var_311_begin_0 = const()[name = string("op_311_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_311_end_0 = const()[name = string("op_311_end_0"), val = tensor<int32, [5]>([512, 4, 2, 8, 8])];
+            tensor<bool, [5]> var_311_end_mask_0 = const()[name = string("op_311_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_311_squeeze_mask_0 = const()[name = string("op_311_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 4, 8, 8]> var_311_cast_fp16 = slice_by_index(begin = var_311_begin_0, end = var_311_end_0, end_mask = var_311_end_mask_0, squeeze_mask = var_311_squeeze_mask_0, x = pair_9_cast_fp16)[name = string("op_311_cast_fp16")];
+            bool var_314_transpose_x_0 = const()[name = string("op_314_transpose_x_0"), val = bool(false)];
+            bool var_314_transpose_y_0 = const()[name = string("op_314_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 8, 8]> var_314_cast_fp16 = matmul(transpose_x = var_314_transpose_x_0, transpose_y = var_314_transpose_y_0, x = x_11_cast_fp16, y = var_308_cast_fp16)[name = string("op_314_cast_fp16")];
+            bool var_315_transpose_x_0 = const()[name = string("op_315_transpose_x_0"), val = bool(false)];
+            bool var_315_transpose_y_0 = const()[name = string("op_315_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 8, 8]> var_315_cast_fp16 = matmul(transpose_x = var_315_transpose_x_0, transpose_y = var_315_transpose_y_0, x = var_311_cast_fp16, y = var_314_cast_fp16)[name = string("op_315_cast_fp16")];
+            fp16 const_9_promoted_to_fp16 = const()[name = string("const_9_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 4, 8, 8]> lower_7_cast_fp16 = mul(x = var_315_cast_fp16, y = const_9_promoted_to_fp16)[name = string("lower_7_cast_fp16")];
+            bool top_7_interleave_0 = const()[name = string("top_7_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 8, 8]> var_317_to_fp16 = const()[name = string("op_317_to_fp16"), val = tensor<fp16, [512, 4, 8, 8]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(682618048)))];
+            tensor<fp16, [512, 4, 8, 16]> top_7_cast_fp16 = concat(axis = var_49, interleave = top_7_interleave_0, values = (var_308_cast_fp16, var_317_to_fp16))[name = string("top_7_cast_fp16")];
+            bool bottom_7_interleave_0 = const()[name = string("bottom_7_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 8, 16]> bottom_7_cast_fp16 = concat(axis = var_49, interleave = bottom_7_interleave_0, values = (lower_7_cast_fp16, var_311_cast_fp16))[name = string("bottom_7_cast_fp16")];
+            bool inv_9_interleave_0 = const()[name = string("inv_9_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 16, 16]> inv_9_cast_fp16 = concat(axis = var_34, interleave = inv_9_interleave_0, values = (top_7_cast_fp16, bottom_7_cast_fp16))[name = string("inv_9_cast_fp16")];
+            tensor<int32, [5]> var_324 = const()[name = string("op_324"), val = tensor<int32, [5]>([512, 2, 32, 2, 32])];
+            tensor<fp16, [512, 2, 32, 2, 32]> var_325_cast_fp16 = reshape(shape = var_324, x = t_3_cast_fp16)[name = string("op_325_cast_fp16")];
+            tensor<fp16, [2, 1, 2, 1]> var_329_to_fp16 = const()[name = string("op_329_to_fp16"), val = tensor<fp16, [2, 1, 2, 1]>([[[[0x1p+0], [0x0p+0]]], [[[0x0p+0], [0x1p+0]]]])];
+            tensor<fp16, [512, 2, 32, 2, 32]> var_330_cast_fp16 = mul(x = var_325_cast_fp16, y = var_329_to_fp16)[name = string("op_330_cast_fp16")];
+            tensor<int32, [1]> blocks_9_axes_0 = const()[name = string("blocks_9_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_9_keep_dims_0 = const()[name = string("blocks_9_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 32, 32]> blocks_9_cast_fp16 = reduce_sum(axes = blocks_9_axes_0, keep_dims = blocks_9_keep_dims_0, x = var_330_cast_fp16)[name = string("blocks_9_cast_fp16")];
+            tensor<int32, [4]> var_333_begin_0 = const()[name = string("op_333_begin_0"), val = tensor<int32, [4]>([0, 0, 16, 0])];
+            tensor<int32, [4]> var_333_end_0 = const()[name = string("op_333_end_0"), val = tensor<int32, [4]>([512, 2, 32, 32])];
+            tensor<bool, [4]> var_333_end_mask_0 = const()[name = string("op_333_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 2, 16, 32]> var_333_cast_fp16 = slice_by_index(begin = var_333_begin_0, end = var_333_end_0, end_mask = var_333_end_mask_0, x = blocks_9_cast_fp16)[name = string("op_333_cast_fp16")];
+            tensor<int32, [4]> x_13_begin_0 = const()[name = string("x_13_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_13_end_0 = const()[name = string("x_13_end_0"), val = tensor<int32, [4]>([512, 2, 16, 16])];
+            tensor<bool, [4]> x_13_end_mask_0 = const()[name = string("x_13_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 2, 16, 16]> x_13_cast_fp16 = slice_by_index(begin = x_13_begin_0, end = x_13_end_0, end_mask = x_13_end_mask_0, x = var_333_cast_fp16)[name = string("x_13_cast_fp16")];
+            tensor<int32, [5]> var_335 = const()[name = string("op_335"), val = tensor<int32, [5]>([512, 2, 2, 16, 16])];
+            tensor<fp16, [512, 2, 2, 16, 16]> pair_11_cast_fp16 = reshape(shape = var_335, x = inv_9_cast_fp16)[name = string("pair_11_cast_fp16")];
+            tensor<int32, [5]> var_337_begin_0 = const()[name = string("op_337_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_337_end_0 = const()[name = string("op_337_end_0"), val = tensor<int32, [5]>([512, 2, 1, 16, 16])];
+            tensor<bool, [5]> var_337_end_mask_0 = const()[name = string("op_337_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_337_squeeze_mask_0 = const()[name = string("op_337_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 2, 16, 16]> var_337_cast_fp16 = slice_by_index(begin = var_337_begin_0, end = var_337_end_0, end_mask = var_337_end_mask_0, squeeze_mask = var_337_squeeze_mask_0, x = pair_11_cast_fp16)[name = string("op_337_cast_fp16")];
+            tensor<int32, [5]> var_340_begin_0 = const()[name = string("op_340_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_340_end_0 = const()[name = string("op_340_end_0"), val = tensor<int32, [5]>([512, 2, 2, 16, 16])];
+            tensor<bool, [5]> var_340_end_mask_0 = const()[name = string("op_340_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_340_squeeze_mask_0 = const()[name = string("op_340_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 2, 16, 16]> var_340_cast_fp16 = slice_by_index(begin = var_340_begin_0, end = var_340_end_0, end_mask = var_340_end_mask_0, squeeze_mask = var_340_squeeze_mask_0, x = pair_11_cast_fp16)[name = string("op_340_cast_fp16")];
+            bool var_343_transpose_x_0 = const()[name = string("op_343_transpose_x_0"), val = bool(false)];
+            bool var_343_transpose_y_0 = const()[name = string("op_343_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 16, 16]> var_343_cast_fp16 = matmul(transpose_x = var_343_transpose_x_0, transpose_y = var_343_transpose_y_0, x = x_13_cast_fp16, y = var_337_cast_fp16)[name = string("op_343_cast_fp16")];
+            bool var_344_transpose_x_0 = const()[name = string("op_344_transpose_x_0"), val = bool(false)];
+            bool var_344_transpose_y_0 = const()[name = string("op_344_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 16, 16]> var_344_cast_fp16 = matmul(transpose_x = var_344_transpose_x_0, transpose_y = var_344_transpose_y_0, x = var_340_cast_fp16, y = var_343_cast_fp16)[name = string("op_344_cast_fp16")];
+            fp16 const_10_promoted_to_fp16 = const()[name = string("const_10_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 2, 16, 16]> lower_9_cast_fp16 = mul(x = var_344_cast_fp16, y = const_10_promoted_to_fp16)[name = string("lower_9_cast_fp16")];
+            bool top_9_interleave_0 = const()[name = string("top_9_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 16, 16]> var_346_to_fp16 = const()[name = string("op_346_to_fp16"), val = tensor<fp16, [512, 2, 16, 16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(682880256)))];
+            tensor<fp16, [512, 2, 16, 32]> top_9_cast_fp16 = concat(axis = var_49, interleave = top_9_interleave_0, values = (var_337_cast_fp16, var_346_to_fp16))[name = string("top_9_cast_fp16")];
+            bool bottom_9_interleave_0 = const()[name = string("bottom_9_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 16, 32]> bottom_9_cast_fp16 = concat(axis = var_49, interleave = bottom_9_interleave_0, values = (lower_9_cast_fp16, var_340_cast_fp16))[name = string("bottom_9_cast_fp16")];
+            bool inv_11_interleave_0 = const()[name = string("inv_11_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 32, 32]> inv_11_cast_fp16 = concat(axis = var_34, interleave = inv_11_interleave_0, values = (top_9_cast_fp16, bottom_9_cast_fp16))[name = string("inv_11_cast_fp16")];
+            tensor<int32, [5]> var_353 = const()[name = string("op_353"), val = tensor<int32, [5]>([512, 1, 64, 1, 64])];
+            tensor<fp16, [512, 1, 64, 1, 64]> var_354_cast_fp16 = reshape(shape = var_353, x = t_3_cast_fp16)[name = string("op_354_cast_fp16")];
+            tensor<int32, [1]> blocks_11_axes_0 = const()[name = string("blocks_11_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_11_keep_dims_0 = const()[name = string("blocks_11_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 64, 64]> blocks_11_cast_fp16 = reduce_sum(axes = blocks_11_axes_0, keep_dims = blocks_11_keep_dims_0, x = var_354_cast_fp16)[name = string("blocks_11_cast_fp16")];
+            tensor<int32, [4]> var_362_begin_0 = const()[name = string("op_362_begin_0"), val = tensor<int32, [4]>([0, 0, 32, 0])];
+            tensor<int32, [4]> var_362_end_0 = const()[name = string("op_362_end_0"), val = tensor<int32, [4]>([512, 1, 64, 64])];
+            tensor<bool, [4]> var_362_end_mask_0 = const()[name = string("op_362_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 1, 32, 64]> var_362_cast_fp16 = slice_by_index(begin = var_362_begin_0, end = var_362_end_0, end_mask = var_362_end_mask_0, x = blocks_11_cast_fp16)[name = string("op_362_cast_fp16")];
+            tensor<int32, [4]> x_15_begin_0 = const()[name = string("x_15_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_15_end_0 = const()[name = string("x_15_end_0"), val = tensor<int32, [4]>([512, 1, 32, 32])];
+            tensor<bool, [4]> x_15_end_mask_0 = const()[name = string("x_15_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 1, 32, 32]> x_15_cast_fp16 = slice_by_index(begin = x_15_begin_0, end = x_15_end_0, end_mask = x_15_end_mask_0, x = var_362_cast_fp16)[name = string("x_15_cast_fp16")];
+            tensor<int32, [5]> var_364 = const()[name = string("op_364"), val = tensor<int32, [5]>([512, 1, 2, 32, 32])];
+            tensor<fp16, [512, 1, 2, 32, 32]> pair_13_cast_fp16 = reshape(shape = var_364, x = inv_11_cast_fp16)[name = string("pair_13_cast_fp16")];
+            tensor<int32, [5]> var_366_begin_0 = const()[name = string("op_366_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_366_end_0 = const()[name = string("op_366_end_0"), val = tensor<int32, [5]>([512, 1, 1, 32, 32])];
+            tensor<bool, [5]> var_366_end_mask_0 = const()[name = string("op_366_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_366_squeeze_mask_0 = const()[name = string("op_366_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 1, 32, 32]> var_366_cast_fp16 = slice_by_index(begin = var_366_begin_0, end = var_366_end_0, end_mask = var_366_end_mask_0, squeeze_mask = var_366_squeeze_mask_0, x = pair_13_cast_fp16)[name = string("op_366_cast_fp16")];
+            tensor<int32, [5]> var_369_begin_0 = const()[name = string("op_369_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_369_end_0 = const()[name = string("op_369_end_0"), val = tensor<int32, [5]>([512, 1, 2, 32, 32])];
+            tensor<bool, [5]> var_369_end_mask_0 = const()[name = string("op_369_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_369_squeeze_mask_0 = const()[name = string("op_369_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 1, 32, 32]> var_369_cast_fp16 = slice_by_index(begin = var_369_begin_0, end = var_369_end_0, end_mask = var_369_end_mask_0, squeeze_mask = var_369_squeeze_mask_0, x = pair_13_cast_fp16)[name = string("op_369_cast_fp16")];
+            bool var_372_transpose_x_0 = const()[name = string("op_372_transpose_x_0"), val = bool(false)];
+            bool var_372_transpose_y_0 = const()[name = string("op_372_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 32, 32]> var_372_cast_fp16 = matmul(transpose_x = var_372_transpose_x_0, transpose_y = var_372_transpose_y_0, x = x_15_cast_fp16, y = var_366_cast_fp16)[name = string("op_372_cast_fp16")];
+            bool var_373_transpose_x_0 = const()[name = string("op_373_transpose_x_0"), val = bool(false)];
+            bool var_373_transpose_y_0 = const()[name = string("op_373_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 32, 32]> var_373_cast_fp16 = matmul(transpose_x = var_373_transpose_x_0, transpose_y = var_373_transpose_y_0, x = var_369_cast_fp16, y = var_372_cast_fp16)[name = string("op_373_cast_fp16")];
+            fp16 const_11_promoted_to_fp16 = const()[name = string("const_11_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 1, 32, 32]> lower_11_cast_fp16 = mul(x = var_373_cast_fp16, y = const_11_promoted_to_fp16)[name = string("lower_11_cast_fp16")];
+            bool top_11_interleave_0 = const()[name = string("top_11_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 32, 32]> var_375_to_fp16 = const()[name = string("op_375_to_fp16"), val = tensor<fp16, [512, 1, 32, 32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(683404608)))];
+            tensor<fp16, [512, 1, 32, 64]> top_11_cast_fp16 = concat(axis = var_49, interleave = top_11_interleave_0, values = (var_366_cast_fp16, var_375_to_fp16))[name = string("top_11_cast_fp16")];
+            bool bottom_11_interleave_0 = const()[name = string("bottom_11_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 32, 64]> bottom_11_cast_fp16 = concat(axis = var_49, interleave = bottom_11_interleave_0, values = (lower_11_cast_fp16, var_369_cast_fp16))[name = string("bottom_11_cast_fp16")];
+            bool inv_13_interleave_0 = const()[name = string("inv_13_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 64, 64]> inv_13_cast_fp16 = concat(axis = var_34, interleave = inv_13_interleave_0, values = (top_11_cast_fp16, bottom_11_cast_fp16))[name = string("inv_13_cast_fp16")];
+            tensor<int32, [4]> var_382 = const()[name = string("op_382"), val = tensor<int32, [4]>([32, 16, 64, 64])];
+            tensor<fp16, [32, 16, 64, 64]> inv_15_cast_fp16 = reshape(shape = var_382, x = inv_13_cast_fp16)[name = string("inv_15_cast_fp16")];
+            bool new_v_1_transpose_x_0 = const()[name = string("new_v_1_transpose_x_0"), val = bool(false)];
+            bool new_v_1_transpose_y_0 = const()[name = string("new_v_1_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 16, 64, 128]> new_v_1_cast_fp16 = matmul(transpose_x = new_v_1_transpose_x_0, transpose_y = new_v_1_transpose_y_0, x = inv_15_cast_fp16, y = v_beta_1_cast_fp16)[name = string("new_v_1_cast_fp16")];
+            tensor<fp16, [32, 16, 64]> var_385_cast_fp16 = exp(x = cum_1_cast_fp16)[name = string("op_385_cast_fp16")];
+            tensor<int32, [1]> var_386_axes_0 = const()[name = string("op_386_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 16, 64, 1]> var_386_cast_fp16 = expand_dims(axes = var_386_axes_0, x = var_385_cast_fp16)[name = string("op_386_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 128]> var_387_cast_fp16 = mul(x = k_beta_1_cast_fp16, y = var_386_cast_fp16)[name = string("op_387_cast_fp16")];
+            bool k_cumdecay_1_transpose_x_0 = const()[name = string("k_cumdecay_1_transpose_x_0"), val = bool(false)];
+            bool k_cumdecay_1_transpose_y_0 = const()[name = string("k_cumdecay_1_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 16, 64, 128]> k_cumdecay_1_cast_fp16 = matmul(transpose_x = k_cumdecay_1_transpose_x_0, transpose_y = k_cumdecay_1_transpose_y_0, x = inv_15_cast_fp16, y = var_387_cast_fp16)[name = string("k_cumdecay_1_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 128]> q_dec_1_cast_fp16 = mul(x = q_9_cast_fp16, y = var_386_cast_fp16)[name = string("q_dec_1_cast_fp16")];
+            tensor<fp16, [32, 16, 64]> var_392_cast_fp16 = exp(x = to_end_1_cast_fp16)[name = string("op_392_cast_fp16")];
+            tensor<int32, [1]> var_393_axes_0 = const()[name = string("op_393_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 16, 64, 1]> var_393_cast_fp16 = expand_dims(axes = var_393_axes_0, x = var_392_cast_fp16)[name = string("op_393_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 128]> k_dec_1_cast_fp16 = mul(x = k_7_cast_fp16, y = var_393_cast_fp16)[name = string("k_dec_1_cast_fp16")];
+            tensor<int32, [3]> var_395_begin_0 = const()[name = string("op_395_begin_0"), val = tensor<int32, [3]>([0, 0, -1])];
+            tensor<int32, [3]> var_395_end_0 = const()[name = string("op_395_end_0"), val = tensor<int32, [3]>([32, 16, 64])];
+            tensor<bool, [3]> var_395_end_mask_0 = const()[name = string("op_395_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
+            tensor<fp16, [32, 16, 1]> var_395_cast_fp16 = slice_by_index(begin = var_395_begin_0, end = var_395_end_0, end_mask = var_395_end_mask_0, x = cum_1_cast_fp16)[name = string("op_395_cast_fp16")];
+            tensor<fp16, [32, 16, 1]> var_396_cast_fp16 = exp(x = var_395_cast_fp16)[name = string("op_396_cast_fp16")];
+            tensor<int32, [1]> chunk_decay_1_axes_0 = const()[name = string("chunk_decay_1_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 16, 1, 1]> chunk_decay_1_cast_fp16 = expand_dims(axes = chunk_decay_1_axes_0, x = var_396_cast_fp16)[name = string("chunk_decay_1_cast_fp16")];
+            tensor<int32, [4]> var_401_begin_0 = const()[name = string("op_401_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_401_end_0 = const()[name = string("op_401_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_401_end_mask_0 = const()[name = string("op_401_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_401_squeeze_mask_0 = const()[name = string("op_401_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_401_cast_fp16 = slice_by_index(begin = var_401_begin_0, end = var_401_end_0, end_mask = var_401_end_mask_0, squeeze_mask = var_401_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_401_cast_fp16")];
+            tensor<int32, [4]> var_403_begin_0 = const()[name = string("op_403_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_403_end_0 = const()[name = string("op_403_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_403_end_mask_0 = const()[name = string("op_403_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_403_squeeze_mask_0 = const()[name = string("op_403_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_403_cast_fp16 = slice_by_index(begin = var_403_begin_0, end = var_403_end_0, end_mask = var_403_end_mask_0, squeeze_mask = var_403_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_403_cast_fp16")];
+            bool var_404_transpose_x_0 = const()[name = string("op_404_transpose_x_0"), val = bool(false)];
+            bool var_404_transpose_y_0 = const()[name = string("op_404_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> state_1_to_fp16 = const()[name = string("state_1_to_fp16"), val = tensor<fp16, [32, 128, 128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(684453248)))];
+            tensor<fp16, [32, 64, 128]> var_404_cast_fp16 = matmul(transpose_x = var_404_transpose_x_0, transpose_y = var_404_transpose_y_0, x = var_403_cast_fp16, y = state_1_to_fp16)[name = string("op_404_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_1_cast_fp16 = sub(x = var_401_cast_fp16, y = var_404_cast_fp16)[name = string("v_new_1_cast_fp16")];
+            tensor<int32, [4]> var_407_begin_0 = const()[name = string("op_407_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_407_end_0 = const()[name = string("op_407_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_407_end_mask_0 = const()[name = string("op_407_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_407_squeeze_mask_0 = const()[name = string("op_407_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_407_cast_fp16 = slice_by_index(begin = var_407_begin_0, end = var_407_end_0, end_mask = var_407_end_mask_0, squeeze_mask = var_407_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_407_cast_fp16")];
+            bool var_408_transpose_x_0 = const()[name = string("op_408_transpose_x_0"), val = bool(false)];
+            bool var_408_transpose_y_0 = const()[name = string("op_408_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_408_cast_fp16 = matmul(transpose_x = var_408_transpose_x_0, transpose_y = var_408_transpose_y_0, x = var_407_cast_fp16, y = state_1_to_fp16)[name = string("op_408_cast_fp16")];
+            tensor<int32, [4]> var_410_begin_0 = const()[name = string("op_410_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_410_end_0 = const()[name = string("op_410_end_0"), val = tensor<int32, [4]>([32, 1, 64, 64])];
+            tensor<bool, [4]> var_410_end_mask_0 = const()[name = string("op_410_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_410_squeeze_mask_0 = const()[name = string("op_410_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_410_cast_fp16 = slice_by_index(begin = var_410_begin_0, end = var_410_end_0, end_mask = var_410_end_mask_0, squeeze_mask = var_410_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_410_cast_fp16")];
+            bool var_411_transpose_x_0 = const()[name = string("op_411_transpose_x_0"), val = bool(false)];
+            bool var_411_transpose_y_0 = const()[name = string("op_411_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_411_cast_fp16 = matmul(transpose_x = var_411_transpose_x_0, transpose_y = var_411_transpose_y_0, x = var_410_cast_fp16, y = v_new_1_cast_fp16)[name = string("op_411_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_412_cast_fp16 = add(x = var_408_cast_fp16, y = var_411_cast_fp16)[name = string("op_412_cast_fp16")];
+            tensor<int32, [4]> var_414_begin_0 = const()[name = string("op_414_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_414_end_0 = const()[name = string("op_414_end_0"), val = tensor<int32, [4]>([32, 1, 1, 1])];
+            tensor<bool, [4]> var_414_end_mask_0 = const()[name = string("op_414_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_414_squeeze_mask_0 = const()[name = string("op_414_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_414_cast_fp16 = slice_by_index(begin = var_414_begin_0, end = var_414_end_0, end_mask = var_414_end_mask_0, squeeze_mask = var_414_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_414_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_415_cast_fp16 = mul(x = state_1_to_fp16, y = var_414_cast_fp16)[name = string("op_415_cast_fp16")];
+            tensor<int32, [4]> var_417_begin_0 = const()[name = string("op_417_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_417_end_0 = const()[name = string("op_417_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_417_end_mask_0 = const()[name = string("op_417_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_417_squeeze_mask_0 = const()[name = string("op_417_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_417_cast_fp16 = slice_by_index(begin = var_417_begin_0, end = var_417_end_0, end_mask = var_417_end_mask_0, squeeze_mask = var_417_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_417_cast_fp16")];
+            bool var_419_transpose_x_1 = const()[name = string("op_419_transpose_x_1"), val = bool(true)];
+            bool var_419_transpose_y_1 = const()[name = string("op_419_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_419_cast_fp16 = matmul(transpose_x = var_419_transpose_x_1, transpose_y = var_419_transpose_y_1, x = var_417_cast_fp16, y = v_new_1_cast_fp16)[name = string("op_419_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_3_cast_fp16 = add(x = var_415_cast_fp16, y = var_419_cast_fp16)[name = string("state_3_cast_fp16")];
+            tensor<int32, [4]> var_422_begin_0 = const()[name = string("op_422_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_422_end_0 = const()[name = string("op_422_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_422_end_mask_0 = const()[name = string("op_422_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_422_squeeze_mask_0 = const()[name = string("op_422_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_422_cast_fp16 = slice_by_index(begin = var_422_begin_0, end = var_422_end_0, end_mask = var_422_end_mask_0, squeeze_mask = var_422_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_422_cast_fp16")];
+            tensor<int32, [4]> var_424_begin_0 = const()[name = string("op_424_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_424_end_0 = const()[name = string("op_424_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_424_end_mask_0 = const()[name = string("op_424_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_424_squeeze_mask_0 = const()[name = string("op_424_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_424_cast_fp16 = slice_by_index(begin = var_424_begin_0, end = var_424_end_0, end_mask = var_424_end_mask_0, squeeze_mask = var_424_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_424_cast_fp16")];
+            bool var_425_transpose_x_0 = const()[name = string("op_425_transpose_x_0"), val = bool(false)];
+            bool var_425_transpose_y_0 = const()[name = string("op_425_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_425_cast_fp16 = matmul(transpose_x = var_425_transpose_x_0, transpose_y = var_425_transpose_y_0, x = var_424_cast_fp16, y = state_3_cast_fp16)[name = string("op_425_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_3_cast_fp16 = sub(x = var_422_cast_fp16, y = var_425_cast_fp16)[name = string("v_new_3_cast_fp16")];
+            tensor<int32, [4]> var_428_begin_0 = const()[name = string("op_428_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_428_end_0 = const()[name = string("op_428_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_428_end_mask_0 = const()[name = string("op_428_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_428_squeeze_mask_0 = const()[name = string("op_428_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_428_cast_fp16 = slice_by_index(begin = var_428_begin_0, end = var_428_end_0, end_mask = var_428_end_mask_0, squeeze_mask = var_428_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_428_cast_fp16")];
+            bool var_429_transpose_x_0 = const()[name = string("op_429_transpose_x_0"), val = bool(false)];
+            bool var_429_transpose_y_0 = const()[name = string("op_429_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_429_cast_fp16 = matmul(transpose_x = var_429_transpose_x_0, transpose_y = var_429_transpose_y_0, x = var_428_cast_fp16, y = state_3_cast_fp16)[name = string("op_429_cast_fp16")];
+            tensor<int32, [4]> var_431_begin_0 = const()[name = string("op_431_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_431_end_0 = const()[name = string("op_431_end_0"), val = tensor<int32, [4]>([32, 2, 64, 64])];
+            tensor<bool, [4]> var_431_end_mask_0 = const()[name = string("op_431_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_431_squeeze_mask_0 = const()[name = string("op_431_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_431_cast_fp16 = slice_by_index(begin = var_431_begin_0, end = var_431_end_0, end_mask = var_431_end_mask_0, squeeze_mask = var_431_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_431_cast_fp16")];
+            bool var_432_transpose_x_0 = const()[name = string("op_432_transpose_x_0"), val = bool(false)];
+            bool var_432_transpose_y_0 = const()[name = string("op_432_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_432_cast_fp16 = matmul(transpose_x = var_432_transpose_x_0, transpose_y = var_432_transpose_y_0, x = var_431_cast_fp16, y = v_new_3_cast_fp16)[name = string("op_432_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_433_cast_fp16 = add(x = var_429_cast_fp16, y = var_432_cast_fp16)[name = string("op_433_cast_fp16")];
+            tensor<int32, [4]> var_435_begin_0 = const()[name = string("op_435_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_435_end_0 = const()[name = string("op_435_end_0"), val = tensor<int32, [4]>([32, 2, 1, 1])];
+            tensor<bool, [4]> var_435_end_mask_0 = const()[name = string("op_435_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_435_squeeze_mask_0 = const()[name = string("op_435_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_435_cast_fp16 = slice_by_index(begin = var_435_begin_0, end = var_435_end_0, end_mask = var_435_end_mask_0, squeeze_mask = var_435_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_435_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_436_cast_fp16 = mul(x = state_3_cast_fp16, y = var_435_cast_fp16)[name = string("op_436_cast_fp16")];
+            tensor<int32, [4]> var_438_begin_0 = const()[name = string("op_438_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_438_end_0 = const()[name = string("op_438_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_438_end_mask_0 = const()[name = string("op_438_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_438_squeeze_mask_0 = const()[name = string("op_438_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_438_cast_fp16 = slice_by_index(begin = var_438_begin_0, end = var_438_end_0, end_mask = var_438_end_mask_0, squeeze_mask = var_438_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_438_cast_fp16")];
+            bool var_440_transpose_x_1 = const()[name = string("op_440_transpose_x_1"), val = bool(true)];
+            bool var_440_transpose_y_1 = const()[name = string("op_440_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_440_cast_fp16 = matmul(transpose_x = var_440_transpose_x_1, transpose_y = var_440_transpose_y_1, x = var_438_cast_fp16, y = v_new_3_cast_fp16)[name = string("op_440_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_5_cast_fp16 = add(x = var_436_cast_fp16, y = var_440_cast_fp16)[name = string("state_5_cast_fp16")];
+            tensor<int32, [4]> var_443_begin_0 = const()[name = string("op_443_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_443_end_0 = const()[name = string("op_443_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_443_end_mask_0 = const()[name = string("op_443_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_443_squeeze_mask_0 = const()[name = string("op_443_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_443_cast_fp16 = slice_by_index(begin = var_443_begin_0, end = var_443_end_0, end_mask = var_443_end_mask_0, squeeze_mask = var_443_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_443_cast_fp16")];
+            tensor<int32, [4]> var_445_begin_0 = const()[name = string("op_445_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_445_end_0 = const()[name = string("op_445_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_445_end_mask_0 = const()[name = string("op_445_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_445_squeeze_mask_0 = const()[name = string("op_445_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_445_cast_fp16 = slice_by_index(begin = var_445_begin_0, end = var_445_end_0, end_mask = var_445_end_mask_0, squeeze_mask = var_445_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_445_cast_fp16")];
+            bool var_446_transpose_x_0 = const()[name = string("op_446_transpose_x_0"), val = bool(false)];
+            bool var_446_transpose_y_0 = const()[name = string("op_446_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_446_cast_fp16 = matmul(transpose_x = var_446_transpose_x_0, transpose_y = var_446_transpose_y_0, x = var_445_cast_fp16, y = state_5_cast_fp16)[name = string("op_446_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_5_cast_fp16 = sub(x = var_443_cast_fp16, y = var_446_cast_fp16)[name = string("v_new_5_cast_fp16")];
+            tensor<int32, [4]> var_449_begin_0 = const()[name = string("op_449_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_449_end_0 = const()[name = string("op_449_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_449_end_mask_0 = const()[name = string("op_449_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_449_squeeze_mask_0 = const()[name = string("op_449_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_449_cast_fp16 = slice_by_index(begin = var_449_begin_0, end = var_449_end_0, end_mask = var_449_end_mask_0, squeeze_mask = var_449_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_449_cast_fp16")];
+            bool var_450_transpose_x_0 = const()[name = string("op_450_transpose_x_0"), val = bool(false)];
+            bool var_450_transpose_y_0 = const()[name = string("op_450_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_450_cast_fp16 = matmul(transpose_x = var_450_transpose_x_0, transpose_y = var_450_transpose_y_0, x = var_449_cast_fp16, y = state_5_cast_fp16)[name = string("op_450_cast_fp16")];
+            tensor<int32, [4]> var_452_begin_0 = const()[name = string("op_452_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_452_end_0 = const()[name = string("op_452_end_0"), val = tensor<int32, [4]>([32, 3, 64, 64])];
+            tensor<bool, [4]> var_452_end_mask_0 = const()[name = string("op_452_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_452_squeeze_mask_0 = const()[name = string("op_452_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_452_cast_fp16 = slice_by_index(begin = var_452_begin_0, end = var_452_end_0, end_mask = var_452_end_mask_0, squeeze_mask = var_452_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_452_cast_fp16")];
+            bool var_453_transpose_x_0 = const()[name = string("op_453_transpose_x_0"), val = bool(false)];
+            bool var_453_transpose_y_0 = const()[name = string("op_453_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_453_cast_fp16 = matmul(transpose_x = var_453_transpose_x_0, transpose_y = var_453_transpose_y_0, x = var_452_cast_fp16, y = v_new_5_cast_fp16)[name = string("op_453_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_454_cast_fp16 = add(x = var_450_cast_fp16, y = var_453_cast_fp16)[name = string("op_454_cast_fp16")];
+            tensor<int32, [4]> var_456_begin_0 = const()[name = string("op_456_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_456_end_0 = const()[name = string("op_456_end_0"), val = tensor<int32, [4]>([32, 3, 1, 1])];
+            tensor<bool, [4]> var_456_end_mask_0 = const()[name = string("op_456_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_456_squeeze_mask_0 = const()[name = string("op_456_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_456_cast_fp16 = slice_by_index(begin = var_456_begin_0, end = var_456_end_0, end_mask = var_456_end_mask_0, squeeze_mask = var_456_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_456_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_457_cast_fp16 = mul(x = state_5_cast_fp16, y = var_456_cast_fp16)[name = string("op_457_cast_fp16")];
+            tensor<int32, [4]> var_459_begin_0 = const()[name = string("op_459_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_459_end_0 = const()[name = string("op_459_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_459_end_mask_0 = const()[name = string("op_459_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_459_squeeze_mask_0 = const()[name = string("op_459_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_459_cast_fp16 = slice_by_index(begin = var_459_begin_0, end = var_459_end_0, end_mask = var_459_end_mask_0, squeeze_mask = var_459_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_459_cast_fp16")];
+            bool var_461_transpose_x_1 = const()[name = string("op_461_transpose_x_1"), val = bool(true)];
+            bool var_461_transpose_y_1 = const()[name = string("op_461_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_461_cast_fp16 = matmul(transpose_x = var_461_transpose_x_1, transpose_y = var_461_transpose_y_1, x = var_459_cast_fp16, y = v_new_5_cast_fp16)[name = string("op_461_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_7_cast_fp16 = add(x = var_457_cast_fp16, y = var_461_cast_fp16)[name = string("state_7_cast_fp16")];
+            tensor<int32, [4]> var_464_begin_0 = const()[name = string("op_464_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_464_end_0 = const()[name = string("op_464_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_464_end_mask_0 = const()[name = string("op_464_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_464_squeeze_mask_0 = const()[name = string("op_464_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_464_cast_fp16 = slice_by_index(begin = var_464_begin_0, end = var_464_end_0, end_mask = var_464_end_mask_0, squeeze_mask = var_464_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_464_cast_fp16")];
+            tensor<int32, [4]> var_466_begin_0 = const()[name = string("op_466_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_466_end_0 = const()[name = string("op_466_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_466_end_mask_0 = const()[name = string("op_466_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_466_squeeze_mask_0 = const()[name = string("op_466_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_466_cast_fp16 = slice_by_index(begin = var_466_begin_0, end = var_466_end_0, end_mask = var_466_end_mask_0, squeeze_mask = var_466_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_466_cast_fp16")];
+            bool var_467_transpose_x_0 = const()[name = string("op_467_transpose_x_0"), val = bool(false)];
+            bool var_467_transpose_y_0 = const()[name = string("op_467_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_467_cast_fp16 = matmul(transpose_x = var_467_transpose_x_0, transpose_y = var_467_transpose_y_0, x = var_466_cast_fp16, y = state_7_cast_fp16)[name = string("op_467_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_7_cast_fp16 = sub(x = var_464_cast_fp16, y = var_467_cast_fp16)[name = string("v_new_7_cast_fp16")];
+            tensor<int32, [4]> var_470_begin_0 = const()[name = string("op_470_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_470_end_0 = const()[name = string("op_470_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_470_end_mask_0 = const()[name = string("op_470_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_470_squeeze_mask_0 = const()[name = string("op_470_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_470_cast_fp16 = slice_by_index(begin = var_470_begin_0, end = var_470_end_0, end_mask = var_470_end_mask_0, squeeze_mask = var_470_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_470_cast_fp16")];
+            bool var_471_transpose_x_0 = const()[name = string("op_471_transpose_x_0"), val = bool(false)];
+            bool var_471_transpose_y_0 = const()[name = string("op_471_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_471_cast_fp16 = matmul(transpose_x = var_471_transpose_x_0, transpose_y = var_471_transpose_y_0, x = var_470_cast_fp16, y = state_7_cast_fp16)[name = string("op_471_cast_fp16")];
+            tensor<int32, [4]> var_473_begin_0 = const()[name = string("op_473_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_473_end_0 = const()[name = string("op_473_end_0"), val = tensor<int32, [4]>([32, 4, 64, 64])];
+            tensor<bool, [4]> var_473_end_mask_0 = const()[name = string("op_473_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_473_squeeze_mask_0 = const()[name = string("op_473_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_473_cast_fp16 = slice_by_index(begin = var_473_begin_0, end = var_473_end_0, end_mask = var_473_end_mask_0, squeeze_mask = var_473_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_473_cast_fp16")];
+            bool var_474_transpose_x_0 = const()[name = string("op_474_transpose_x_0"), val = bool(false)];
+            bool var_474_transpose_y_0 = const()[name = string("op_474_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_474_cast_fp16 = matmul(transpose_x = var_474_transpose_x_0, transpose_y = var_474_transpose_y_0, x = var_473_cast_fp16, y = v_new_7_cast_fp16)[name = string("op_474_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_475_cast_fp16 = add(x = var_471_cast_fp16, y = var_474_cast_fp16)[name = string("op_475_cast_fp16")];
+            tensor<int32, [4]> var_477_begin_0 = const()[name = string("op_477_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_477_end_0 = const()[name = string("op_477_end_0"), val = tensor<int32, [4]>([32, 4, 1, 1])];
+            tensor<bool, [4]> var_477_end_mask_0 = const()[name = string("op_477_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_477_squeeze_mask_0 = const()[name = string("op_477_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_477_cast_fp16 = slice_by_index(begin = var_477_begin_0, end = var_477_end_0, end_mask = var_477_end_mask_0, squeeze_mask = var_477_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_477_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_478_cast_fp16 = mul(x = state_7_cast_fp16, y = var_477_cast_fp16)[name = string("op_478_cast_fp16")];
+            tensor<int32, [4]> var_480_begin_0 = const()[name = string("op_480_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_480_end_0 = const()[name = string("op_480_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_480_end_mask_0 = const()[name = string("op_480_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_480_squeeze_mask_0 = const()[name = string("op_480_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_480_cast_fp16 = slice_by_index(begin = var_480_begin_0, end = var_480_end_0, end_mask = var_480_end_mask_0, squeeze_mask = var_480_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_480_cast_fp16")];
+            bool var_482_transpose_x_1 = const()[name = string("op_482_transpose_x_1"), val = bool(true)];
+            bool var_482_transpose_y_1 = const()[name = string("op_482_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_482_cast_fp16 = matmul(transpose_x = var_482_transpose_x_1, transpose_y = var_482_transpose_y_1, x = var_480_cast_fp16, y = v_new_7_cast_fp16)[name = string("op_482_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_9_cast_fp16 = add(x = var_478_cast_fp16, y = var_482_cast_fp16)[name = string("state_9_cast_fp16")];
+            tensor<int32, [4]> var_485_begin_0 = const()[name = string("op_485_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_485_end_0 = const()[name = string("op_485_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_485_end_mask_0 = const()[name = string("op_485_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_485_squeeze_mask_0 = const()[name = string("op_485_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_485_cast_fp16 = slice_by_index(begin = var_485_begin_0, end = var_485_end_0, end_mask = var_485_end_mask_0, squeeze_mask = var_485_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_485_cast_fp16")];
+            tensor<int32, [4]> var_487_begin_0 = const()[name = string("op_487_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_487_end_0 = const()[name = string("op_487_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_487_end_mask_0 = const()[name = string("op_487_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_487_squeeze_mask_0 = const()[name = string("op_487_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_487_cast_fp16 = slice_by_index(begin = var_487_begin_0, end = var_487_end_0, end_mask = var_487_end_mask_0, squeeze_mask = var_487_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_487_cast_fp16")];
+            bool var_488_transpose_x_0 = const()[name = string("op_488_transpose_x_0"), val = bool(false)];
+            bool var_488_transpose_y_0 = const()[name = string("op_488_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_488_cast_fp16 = matmul(transpose_x = var_488_transpose_x_0, transpose_y = var_488_transpose_y_0, x = var_487_cast_fp16, y = state_9_cast_fp16)[name = string("op_488_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_9_cast_fp16 = sub(x = var_485_cast_fp16, y = var_488_cast_fp16)[name = string("v_new_9_cast_fp16")];
+            tensor<int32, [4]> var_491_begin_0 = const()[name = string("op_491_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_491_end_0 = const()[name = string("op_491_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_491_end_mask_0 = const()[name = string("op_491_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_491_squeeze_mask_0 = const()[name = string("op_491_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_491_cast_fp16 = slice_by_index(begin = var_491_begin_0, end = var_491_end_0, end_mask = var_491_end_mask_0, squeeze_mask = var_491_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_491_cast_fp16")];
+            bool var_492_transpose_x_0 = const()[name = string("op_492_transpose_x_0"), val = bool(false)];
+            bool var_492_transpose_y_0 = const()[name = string("op_492_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_492_cast_fp16 = matmul(transpose_x = var_492_transpose_x_0, transpose_y = var_492_transpose_y_0, x = var_491_cast_fp16, y = state_9_cast_fp16)[name = string("op_492_cast_fp16")];
+            tensor<int32, [4]> var_494_begin_0 = const()[name = string("op_494_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_494_end_0 = const()[name = string("op_494_end_0"), val = tensor<int32, [4]>([32, 5, 64, 64])];
+            tensor<bool, [4]> var_494_end_mask_0 = const()[name = string("op_494_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_494_squeeze_mask_0 = const()[name = string("op_494_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_494_cast_fp16 = slice_by_index(begin = var_494_begin_0, end = var_494_end_0, end_mask = var_494_end_mask_0, squeeze_mask = var_494_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_494_cast_fp16")];
+            bool var_495_transpose_x_0 = const()[name = string("op_495_transpose_x_0"), val = bool(false)];
+            bool var_495_transpose_y_0 = const()[name = string("op_495_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_495_cast_fp16 = matmul(transpose_x = var_495_transpose_x_0, transpose_y = var_495_transpose_y_0, x = var_494_cast_fp16, y = v_new_9_cast_fp16)[name = string("op_495_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_496_cast_fp16 = add(x = var_492_cast_fp16, y = var_495_cast_fp16)[name = string("op_496_cast_fp16")];
+            tensor<int32, [4]> var_498_begin_0 = const()[name = string("op_498_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_498_end_0 = const()[name = string("op_498_end_0"), val = tensor<int32, [4]>([32, 5, 1, 1])];
+            tensor<bool, [4]> var_498_end_mask_0 = const()[name = string("op_498_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_498_squeeze_mask_0 = const()[name = string("op_498_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_498_cast_fp16 = slice_by_index(begin = var_498_begin_0, end = var_498_end_0, end_mask = var_498_end_mask_0, squeeze_mask = var_498_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_498_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_499_cast_fp16 = mul(x = state_9_cast_fp16, y = var_498_cast_fp16)[name = string("op_499_cast_fp16")];
+            tensor<int32, [4]> var_501_begin_0 = const()[name = string("op_501_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_501_end_0 = const()[name = string("op_501_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_501_end_mask_0 = const()[name = string("op_501_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_501_squeeze_mask_0 = const()[name = string("op_501_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_501_cast_fp16 = slice_by_index(begin = var_501_begin_0, end = var_501_end_0, end_mask = var_501_end_mask_0, squeeze_mask = var_501_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_501_cast_fp16")];
+            bool var_503_transpose_x_1 = const()[name = string("op_503_transpose_x_1"), val = bool(true)];
+            bool var_503_transpose_y_1 = const()[name = string("op_503_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_503_cast_fp16 = matmul(transpose_x = var_503_transpose_x_1, transpose_y = var_503_transpose_y_1, x = var_501_cast_fp16, y = v_new_9_cast_fp16)[name = string("op_503_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_11_cast_fp16 = add(x = var_499_cast_fp16, y = var_503_cast_fp16)[name = string("state_11_cast_fp16")];
+            tensor<int32, [4]> var_506_begin_0 = const()[name = string("op_506_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_506_end_0 = const()[name = string("op_506_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_506_end_mask_0 = const()[name = string("op_506_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_506_squeeze_mask_0 = const()[name = string("op_506_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_506_cast_fp16 = slice_by_index(begin = var_506_begin_0, end = var_506_end_0, end_mask = var_506_end_mask_0, squeeze_mask = var_506_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_506_cast_fp16")];
+            tensor<int32, [4]> var_508_begin_0 = const()[name = string("op_508_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_508_end_0 = const()[name = string("op_508_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_508_end_mask_0 = const()[name = string("op_508_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_508_squeeze_mask_0 = const()[name = string("op_508_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_508_cast_fp16 = slice_by_index(begin = var_508_begin_0, end = var_508_end_0, end_mask = var_508_end_mask_0, squeeze_mask = var_508_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_508_cast_fp16")];
+            bool var_509_transpose_x_0 = const()[name = string("op_509_transpose_x_0"), val = bool(false)];
+            bool var_509_transpose_y_0 = const()[name = string("op_509_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_509_cast_fp16 = matmul(transpose_x = var_509_transpose_x_0, transpose_y = var_509_transpose_y_0, x = var_508_cast_fp16, y = state_11_cast_fp16)[name = string("op_509_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_11_cast_fp16 = sub(x = var_506_cast_fp16, y = var_509_cast_fp16)[name = string("v_new_11_cast_fp16")];
+            tensor<int32, [4]> var_512_begin_0 = const()[name = string("op_512_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_512_end_0 = const()[name = string("op_512_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_512_end_mask_0 = const()[name = string("op_512_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_512_squeeze_mask_0 = const()[name = string("op_512_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_512_cast_fp16 = slice_by_index(begin = var_512_begin_0, end = var_512_end_0, end_mask = var_512_end_mask_0, squeeze_mask = var_512_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_512_cast_fp16")];
+            bool var_513_transpose_x_0 = const()[name = string("op_513_transpose_x_0"), val = bool(false)];
+            bool var_513_transpose_y_0 = const()[name = string("op_513_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_513_cast_fp16 = matmul(transpose_x = var_513_transpose_x_0, transpose_y = var_513_transpose_y_0, x = var_512_cast_fp16, y = state_11_cast_fp16)[name = string("op_513_cast_fp16")];
+            tensor<int32, [4]> var_515_begin_0 = const()[name = string("op_515_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_515_end_0 = const()[name = string("op_515_end_0"), val = tensor<int32, [4]>([32, 6, 64, 64])];
+            tensor<bool, [4]> var_515_end_mask_0 = const()[name = string("op_515_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_515_squeeze_mask_0 = const()[name = string("op_515_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_515_cast_fp16 = slice_by_index(begin = var_515_begin_0, end = var_515_end_0, end_mask = var_515_end_mask_0, squeeze_mask = var_515_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_515_cast_fp16")];
+            bool var_516_transpose_x_0 = const()[name = string("op_516_transpose_x_0"), val = bool(false)];
+            bool var_516_transpose_y_0 = const()[name = string("op_516_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_516_cast_fp16 = matmul(transpose_x = var_516_transpose_x_0, transpose_y = var_516_transpose_y_0, x = var_515_cast_fp16, y = v_new_11_cast_fp16)[name = string("op_516_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_517_cast_fp16 = add(x = var_513_cast_fp16, y = var_516_cast_fp16)[name = string("op_517_cast_fp16")];
+            tensor<int32, [4]> var_519_begin_0 = const()[name = string("op_519_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_519_end_0 = const()[name = string("op_519_end_0"), val = tensor<int32, [4]>([32, 6, 1, 1])];
+            tensor<bool, [4]> var_519_end_mask_0 = const()[name = string("op_519_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_519_squeeze_mask_0 = const()[name = string("op_519_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_519_cast_fp16 = slice_by_index(begin = var_519_begin_0, end = var_519_end_0, end_mask = var_519_end_mask_0, squeeze_mask = var_519_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_519_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_520_cast_fp16 = mul(x = state_11_cast_fp16, y = var_519_cast_fp16)[name = string("op_520_cast_fp16")];
+            tensor<int32, [4]> var_522_begin_0 = const()[name = string("op_522_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_522_end_0 = const()[name = string("op_522_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_522_end_mask_0 = const()[name = string("op_522_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_522_squeeze_mask_0 = const()[name = string("op_522_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_522_cast_fp16 = slice_by_index(begin = var_522_begin_0, end = var_522_end_0, end_mask = var_522_end_mask_0, squeeze_mask = var_522_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_522_cast_fp16")];
+            bool var_524_transpose_x_1 = const()[name = string("op_524_transpose_x_1"), val = bool(true)];
+            bool var_524_transpose_y_1 = const()[name = string("op_524_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_524_cast_fp16 = matmul(transpose_x = var_524_transpose_x_1, transpose_y = var_524_transpose_y_1, x = var_522_cast_fp16, y = v_new_11_cast_fp16)[name = string("op_524_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_13_cast_fp16 = add(x = var_520_cast_fp16, y = var_524_cast_fp16)[name = string("state_13_cast_fp16")];
+            tensor<int32, [4]> var_527_begin_0 = const()[name = string("op_527_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_527_end_0 = const()[name = string("op_527_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_527_end_mask_0 = const()[name = string("op_527_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_527_squeeze_mask_0 = const()[name = string("op_527_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_527_cast_fp16 = slice_by_index(begin = var_527_begin_0, end = var_527_end_0, end_mask = var_527_end_mask_0, squeeze_mask = var_527_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_527_cast_fp16")];
+            tensor<int32, [4]> var_529_begin_0 = const()[name = string("op_529_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_529_end_0 = const()[name = string("op_529_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_529_end_mask_0 = const()[name = string("op_529_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_529_squeeze_mask_0 = const()[name = string("op_529_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_529_cast_fp16 = slice_by_index(begin = var_529_begin_0, end = var_529_end_0, end_mask = var_529_end_mask_0, squeeze_mask = var_529_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_529_cast_fp16")];
+            bool var_530_transpose_x_0 = const()[name = string("op_530_transpose_x_0"), val = bool(false)];
+            bool var_530_transpose_y_0 = const()[name = string("op_530_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_530_cast_fp16 = matmul(transpose_x = var_530_transpose_x_0, transpose_y = var_530_transpose_y_0, x = var_529_cast_fp16, y = state_13_cast_fp16)[name = string("op_530_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_13_cast_fp16 = sub(x = var_527_cast_fp16, y = var_530_cast_fp16)[name = string("v_new_13_cast_fp16")];
+            tensor<int32, [4]> var_533_begin_0 = const()[name = string("op_533_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_533_end_0 = const()[name = string("op_533_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_533_end_mask_0 = const()[name = string("op_533_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_533_squeeze_mask_0 = const()[name = string("op_533_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_533_cast_fp16 = slice_by_index(begin = var_533_begin_0, end = var_533_end_0, end_mask = var_533_end_mask_0, squeeze_mask = var_533_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_533_cast_fp16")];
+            bool var_534_transpose_x_0 = const()[name = string("op_534_transpose_x_0"), val = bool(false)];
+            bool var_534_transpose_y_0 = const()[name = string("op_534_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_534_cast_fp16 = matmul(transpose_x = var_534_transpose_x_0, transpose_y = var_534_transpose_y_0, x = var_533_cast_fp16, y = state_13_cast_fp16)[name = string("op_534_cast_fp16")];
+            tensor<int32, [4]> var_536_begin_0 = const()[name = string("op_536_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_536_end_0 = const()[name = string("op_536_end_0"), val = tensor<int32, [4]>([32, 7, 64, 64])];
+            tensor<bool, [4]> var_536_end_mask_0 = const()[name = string("op_536_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_536_squeeze_mask_0 = const()[name = string("op_536_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_536_cast_fp16 = slice_by_index(begin = var_536_begin_0, end = var_536_end_0, end_mask = var_536_end_mask_0, squeeze_mask = var_536_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_536_cast_fp16")];
+            bool var_537_transpose_x_0 = const()[name = string("op_537_transpose_x_0"), val = bool(false)];
+            bool var_537_transpose_y_0 = const()[name = string("op_537_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_537_cast_fp16 = matmul(transpose_x = var_537_transpose_x_0, transpose_y = var_537_transpose_y_0, x = var_536_cast_fp16, y = v_new_13_cast_fp16)[name = string("op_537_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_538_cast_fp16 = add(x = var_534_cast_fp16, y = var_537_cast_fp16)[name = string("op_538_cast_fp16")];
+            tensor<int32, [4]> var_540_begin_0 = const()[name = string("op_540_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_540_end_0 = const()[name = string("op_540_end_0"), val = tensor<int32, [4]>([32, 7, 1, 1])];
+            tensor<bool, [4]> var_540_end_mask_0 = const()[name = string("op_540_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_540_squeeze_mask_0 = const()[name = string("op_540_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_540_cast_fp16 = slice_by_index(begin = var_540_begin_0, end = var_540_end_0, end_mask = var_540_end_mask_0, squeeze_mask = var_540_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_540_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_541_cast_fp16 = mul(x = state_13_cast_fp16, y = var_540_cast_fp16)[name = string("op_541_cast_fp16")];
+            tensor<int32, [4]> var_543_begin_0 = const()[name = string("op_543_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_543_end_0 = const()[name = string("op_543_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_543_end_mask_0 = const()[name = string("op_543_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_543_squeeze_mask_0 = const()[name = string("op_543_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_543_cast_fp16 = slice_by_index(begin = var_543_begin_0, end = var_543_end_0, end_mask = var_543_end_mask_0, squeeze_mask = var_543_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_543_cast_fp16")];
+            bool var_545_transpose_x_1 = const()[name = string("op_545_transpose_x_1"), val = bool(true)];
+            bool var_545_transpose_y_1 = const()[name = string("op_545_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_545_cast_fp16 = matmul(transpose_x = var_545_transpose_x_1, transpose_y = var_545_transpose_y_1, x = var_543_cast_fp16, y = v_new_13_cast_fp16)[name = string("op_545_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_15_cast_fp16 = add(x = var_541_cast_fp16, y = var_545_cast_fp16)[name = string("state_15_cast_fp16")];
+            tensor<int32, [4]> var_548_begin_0 = const()[name = string("op_548_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_548_end_0 = const()[name = string("op_548_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_548_end_mask_0 = const()[name = string("op_548_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_548_squeeze_mask_0 = const()[name = string("op_548_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_548_cast_fp16 = slice_by_index(begin = var_548_begin_0, end = var_548_end_0, end_mask = var_548_end_mask_0, squeeze_mask = var_548_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_548_cast_fp16")];
+            tensor<int32, [4]> var_550_begin_0 = const()[name = string("op_550_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_550_end_0 = const()[name = string("op_550_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_550_end_mask_0 = const()[name = string("op_550_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_550_squeeze_mask_0 = const()[name = string("op_550_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_550_cast_fp16 = slice_by_index(begin = var_550_begin_0, end = var_550_end_0, end_mask = var_550_end_mask_0, squeeze_mask = var_550_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_550_cast_fp16")];
+            bool var_551_transpose_x_0 = const()[name = string("op_551_transpose_x_0"), val = bool(false)];
+            bool var_551_transpose_y_0 = const()[name = string("op_551_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_551_cast_fp16 = matmul(transpose_x = var_551_transpose_x_0, transpose_y = var_551_transpose_y_0, x = var_550_cast_fp16, y = state_15_cast_fp16)[name = string("op_551_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_15_cast_fp16 = sub(x = var_548_cast_fp16, y = var_551_cast_fp16)[name = string("v_new_15_cast_fp16")];
+            tensor<int32, [4]> var_554_begin_0 = const()[name = string("op_554_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_554_end_0 = const()[name = string("op_554_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_554_end_mask_0 = const()[name = string("op_554_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_554_squeeze_mask_0 = const()[name = string("op_554_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_554_cast_fp16 = slice_by_index(begin = var_554_begin_0, end = var_554_end_0, end_mask = var_554_end_mask_0, squeeze_mask = var_554_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_554_cast_fp16")];
+            bool var_555_transpose_x_0 = const()[name = string("op_555_transpose_x_0"), val = bool(false)];
+            bool var_555_transpose_y_0 = const()[name = string("op_555_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_555_cast_fp16 = matmul(transpose_x = var_555_transpose_x_0, transpose_y = var_555_transpose_y_0, x = var_554_cast_fp16, y = state_15_cast_fp16)[name = string("op_555_cast_fp16")];
+            tensor<int32, [4]> var_557_begin_0 = const()[name = string("op_557_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_557_end_0 = const()[name = string("op_557_end_0"), val = tensor<int32, [4]>([32, 8, 64, 64])];
+            tensor<bool, [4]> var_557_end_mask_0 = const()[name = string("op_557_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_557_squeeze_mask_0 = const()[name = string("op_557_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_557_cast_fp16 = slice_by_index(begin = var_557_begin_0, end = var_557_end_0, end_mask = var_557_end_mask_0, squeeze_mask = var_557_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_557_cast_fp16")];
+            bool var_558_transpose_x_0 = const()[name = string("op_558_transpose_x_0"), val = bool(false)];
+            bool var_558_transpose_y_0 = const()[name = string("op_558_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_558_cast_fp16 = matmul(transpose_x = var_558_transpose_x_0, transpose_y = var_558_transpose_y_0, x = var_557_cast_fp16, y = v_new_15_cast_fp16)[name = string("op_558_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_559_cast_fp16 = add(x = var_555_cast_fp16, y = var_558_cast_fp16)[name = string("op_559_cast_fp16")];
+            tensor<int32, [4]> var_561_begin_0 = const()[name = string("op_561_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_561_end_0 = const()[name = string("op_561_end_0"), val = tensor<int32, [4]>([32, 8, 1, 1])];
+            tensor<bool, [4]> var_561_end_mask_0 = const()[name = string("op_561_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_561_squeeze_mask_0 = const()[name = string("op_561_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_561_cast_fp16 = slice_by_index(begin = var_561_begin_0, end = var_561_end_0, end_mask = var_561_end_mask_0, squeeze_mask = var_561_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_561_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_562_cast_fp16 = mul(x = state_15_cast_fp16, y = var_561_cast_fp16)[name = string("op_562_cast_fp16")];
+            tensor<int32, [4]> var_564_begin_0 = const()[name = string("op_564_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_564_end_0 = const()[name = string("op_564_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_564_end_mask_0 = const()[name = string("op_564_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_564_squeeze_mask_0 = const()[name = string("op_564_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_564_cast_fp16 = slice_by_index(begin = var_564_begin_0, end = var_564_end_0, end_mask = var_564_end_mask_0, squeeze_mask = var_564_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_564_cast_fp16")];
+            bool var_566_transpose_x_1 = const()[name = string("op_566_transpose_x_1"), val = bool(true)];
+            bool var_566_transpose_y_1 = const()[name = string("op_566_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_566_cast_fp16 = matmul(transpose_x = var_566_transpose_x_1, transpose_y = var_566_transpose_y_1, x = var_564_cast_fp16, y = v_new_15_cast_fp16)[name = string("op_566_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_17_cast_fp16 = add(x = var_562_cast_fp16, y = var_566_cast_fp16)[name = string("state_17_cast_fp16")];
+            tensor<int32, [4]> var_569_begin_0 = const()[name = string("op_569_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_569_end_0 = const()[name = string("op_569_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_569_end_mask_0 = const()[name = string("op_569_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_569_squeeze_mask_0 = const()[name = string("op_569_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_569_cast_fp16 = slice_by_index(begin = var_569_begin_0, end = var_569_end_0, end_mask = var_569_end_mask_0, squeeze_mask = var_569_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_569_cast_fp16")];
+            tensor<int32, [4]> var_571_begin_0 = const()[name = string("op_571_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_571_end_0 = const()[name = string("op_571_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_571_end_mask_0 = const()[name = string("op_571_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_571_squeeze_mask_0 = const()[name = string("op_571_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_571_cast_fp16 = slice_by_index(begin = var_571_begin_0, end = var_571_end_0, end_mask = var_571_end_mask_0, squeeze_mask = var_571_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_571_cast_fp16")];
+            bool var_572_transpose_x_0 = const()[name = string("op_572_transpose_x_0"), val = bool(false)];
+            bool var_572_transpose_y_0 = const()[name = string("op_572_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_572_cast_fp16 = matmul(transpose_x = var_572_transpose_x_0, transpose_y = var_572_transpose_y_0, x = var_571_cast_fp16, y = state_17_cast_fp16)[name = string("op_572_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_17_cast_fp16 = sub(x = var_569_cast_fp16, y = var_572_cast_fp16)[name = string("v_new_17_cast_fp16")];
+            tensor<int32, [4]> var_575_begin_0 = const()[name = string("op_575_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_575_end_0 = const()[name = string("op_575_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_575_end_mask_0 = const()[name = string("op_575_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_575_squeeze_mask_0 = const()[name = string("op_575_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_575_cast_fp16 = slice_by_index(begin = var_575_begin_0, end = var_575_end_0, end_mask = var_575_end_mask_0, squeeze_mask = var_575_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_575_cast_fp16")];
+            bool var_576_transpose_x_0 = const()[name = string("op_576_transpose_x_0"), val = bool(false)];
+            bool var_576_transpose_y_0 = const()[name = string("op_576_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_576_cast_fp16 = matmul(transpose_x = var_576_transpose_x_0, transpose_y = var_576_transpose_y_0, x = var_575_cast_fp16, y = state_17_cast_fp16)[name = string("op_576_cast_fp16")];
+            tensor<int32, [4]> var_578_begin_0 = const()[name = string("op_578_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_578_end_0 = const()[name = string("op_578_end_0"), val = tensor<int32, [4]>([32, 9, 64, 64])];
+            tensor<bool, [4]> var_578_end_mask_0 = const()[name = string("op_578_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_578_squeeze_mask_0 = const()[name = string("op_578_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_578_cast_fp16 = slice_by_index(begin = var_578_begin_0, end = var_578_end_0, end_mask = var_578_end_mask_0, squeeze_mask = var_578_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_578_cast_fp16")];
+            bool var_579_transpose_x_0 = const()[name = string("op_579_transpose_x_0"), val = bool(false)];
+            bool var_579_transpose_y_0 = const()[name = string("op_579_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_579_cast_fp16 = matmul(transpose_x = var_579_transpose_x_0, transpose_y = var_579_transpose_y_0, x = var_578_cast_fp16, y = v_new_17_cast_fp16)[name = string("op_579_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_580_cast_fp16 = add(x = var_576_cast_fp16, y = var_579_cast_fp16)[name = string("op_580_cast_fp16")];
+            tensor<int32, [4]> var_582_begin_0 = const()[name = string("op_582_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_582_end_0 = const()[name = string("op_582_end_0"), val = tensor<int32, [4]>([32, 9, 1, 1])];
+            tensor<bool, [4]> var_582_end_mask_0 = const()[name = string("op_582_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_582_squeeze_mask_0 = const()[name = string("op_582_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_582_cast_fp16 = slice_by_index(begin = var_582_begin_0, end = var_582_end_0, end_mask = var_582_end_mask_0, squeeze_mask = var_582_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_582_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_583_cast_fp16 = mul(x = state_17_cast_fp16, y = var_582_cast_fp16)[name = string("op_583_cast_fp16")];
+            tensor<int32, [4]> var_585_begin_0 = const()[name = string("op_585_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_585_end_0 = const()[name = string("op_585_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_585_end_mask_0 = const()[name = string("op_585_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_585_squeeze_mask_0 = const()[name = string("op_585_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_585_cast_fp16 = slice_by_index(begin = var_585_begin_0, end = var_585_end_0, end_mask = var_585_end_mask_0, squeeze_mask = var_585_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_585_cast_fp16")];
+            bool var_587_transpose_x_1 = const()[name = string("op_587_transpose_x_1"), val = bool(true)];
+            bool var_587_transpose_y_1 = const()[name = string("op_587_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_587_cast_fp16 = matmul(transpose_x = var_587_transpose_x_1, transpose_y = var_587_transpose_y_1, x = var_585_cast_fp16, y = v_new_17_cast_fp16)[name = string("op_587_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_19_cast_fp16 = add(x = var_583_cast_fp16, y = var_587_cast_fp16)[name = string("state_19_cast_fp16")];
+            tensor<int32, [4]> var_590_begin_0 = const()[name = string("op_590_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_590_end_0 = const()[name = string("op_590_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_590_end_mask_0 = const()[name = string("op_590_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_590_squeeze_mask_0 = const()[name = string("op_590_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_590_cast_fp16 = slice_by_index(begin = var_590_begin_0, end = var_590_end_0, end_mask = var_590_end_mask_0, squeeze_mask = var_590_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_590_cast_fp16")];
+            tensor<int32, [4]> var_592_begin_0 = const()[name = string("op_592_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_592_end_0 = const()[name = string("op_592_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_592_end_mask_0 = const()[name = string("op_592_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_592_squeeze_mask_0 = const()[name = string("op_592_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_592_cast_fp16 = slice_by_index(begin = var_592_begin_0, end = var_592_end_0, end_mask = var_592_end_mask_0, squeeze_mask = var_592_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_592_cast_fp16")];
+            bool var_593_transpose_x_0 = const()[name = string("op_593_transpose_x_0"), val = bool(false)];
+            bool var_593_transpose_y_0 = const()[name = string("op_593_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_593_cast_fp16 = matmul(transpose_x = var_593_transpose_x_0, transpose_y = var_593_transpose_y_0, x = var_592_cast_fp16, y = state_19_cast_fp16)[name = string("op_593_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_19_cast_fp16 = sub(x = var_590_cast_fp16, y = var_593_cast_fp16)[name = string("v_new_19_cast_fp16")];
+            tensor<int32, [4]> var_596_begin_0 = const()[name = string("op_596_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_596_end_0 = const()[name = string("op_596_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_596_end_mask_0 = const()[name = string("op_596_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_596_squeeze_mask_0 = const()[name = string("op_596_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_596_cast_fp16 = slice_by_index(begin = var_596_begin_0, end = var_596_end_0, end_mask = var_596_end_mask_0, squeeze_mask = var_596_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_596_cast_fp16")];
+            bool var_597_transpose_x_0 = const()[name = string("op_597_transpose_x_0"), val = bool(false)];
+            bool var_597_transpose_y_0 = const()[name = string("op_597_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_597_cast_fp16 = matmul(transpose_x = var_597_transpose_x_0, transpose_y = var_597_transpose_y_0, x = var_596_cast_fp16, y = state_19_cast_fp16)[name = string("op_597_cast_fp16")];
+            tensor<int32, [4]> var_599_begin_0 = const()[name = string("op_599_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_599_end_0 = const()[name = string("op_599_end_0"), val = tensor<int32, [4]>([32, 10, 64, 64])];
+            tensor<bool, [4]> var_599_end_mask_0 = const()[name = string("op_599_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_599_squeeze_mask_0 = const()[name = string("op_599_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_599_cast_fp16 = slice_by_index(begin = var_599_begin_0, end = var_599_end_0, end_mask = var_599_end_mask_0, squeeze_mask = var_599_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_599_cast_fp16")];
+            bool var_600_transpose_x_0 = const()[name = string("op_600_transpose_x_0"), val = bool(false)];
+            bool var_600_transpose_y_0 = const()[name = string("op_600_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_600_cast_fp16 = matmul(transpose_x = var_600_transpose_x_0, transpose_y = var_600_transpose_y_0, x = var_599_cast_fp16, y = v_new_19_cast_fp16)[name = string("op_600_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_601_cast_fp16 = add(x = var_597_cast_fp16, y = var_600_cast_fp16)[name = string("op_601_cast_fp16")];
+            tensor<int32, [4]> var_603_begin_0 = const()[name = string("op_603_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_603_end_0 = const()[name = string("op_603_end_0"), val = tensor<int32, [4]>([32, 10, 1, 1])];
+            tensor<bool, [4]> var_603_end_mask_0 = const()[name = string("op_603_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_603_squeeze_mask_0 = const()[name = string("op_603_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_603_cast_fp16 = slice_by_index(begin = var_603_begin_0, end = var_603_end_0, end_mask = var_603_end_mask_0, squeeze_mask = var_603_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_603_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_604_cast_fp16 = mul(x = state_19_cast_fp16, y = var_603_cast_fp16)[name = string("op_604_cast_fp16")];
+            tensor<int32, [4]> var_606_begin_0 = const()[name = string("op_606_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_606_end_0 = const()[name = string("op_606_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_606_end_mask_0 = const()[name = string("op_606_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_606_squeeze_mask_0 = const()[name = string("op_606_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_606_cast_fp16 = slice_by_index(begin = var_606_begin_0, end = var_606_end_0, end_mask = var_606_end_mask_0, squeeze_mask = var_606_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_606_cast_fp16")];
+            bool var_608_transpose_x_1 = const()[name = string("op_608_transpose_x_1"), val = bool(true)];
+            bool var_608_transpose_y_1 = const()[name = string("op_608_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_608_cast_fp16 = matmul(transpose_x = var_608_transpose_x_1, transpose_y = var_608_transpose_y_1, x = var_606_cast_fp16, y = v_new_19_cast_fp16)[name = string("op_608_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_21_cast_fp16 = add(x = var_604_cast_fp16, y = var_608_cast_fp16)[name = string("state_21_cast_fp16")];
+            tensor<int32, [4]> var_611_begin_0 = const()[name = string("op_611_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_611_end_0 = const()[name = string("op_611_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_611_end_mask_0 = const()[name = string("op_611_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_611_squeeze_mask_0 = const()[name = string("op_611_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_611_cast_fp16 = slice_by_index(begin = var_611_begin_0, end = var_611_end_0, end_mask = var_611_end_mask_0, squeeze_mask = var_611_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_611_cast_fp16")];
+            tensor<int32, [4]> var_613_begin_0 = const()[name = string("op_613_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_613_end_0 = const()[name = string("op_613_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_613_end_mask_0 = const()[name = string("op_613_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_613_squeeze_mask_0 = const()[name = string("op_613_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_613_cast_fp16 = slice_by_index(begin = var_613_begin_0, end = var_613_end_0, end_mask = var_613_end_mask_0, squeeze_mask = var_613_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_613_cast_fp16")];
+            bool var_614_transpose_x_0 = const()[name = string("op_614_transpose_x_0"), val = bool(false)];
+            bool var_614_transpose_y_0 = const()[name = string("op_614_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_614_cast_fp16 = matmul(transpose_x = var_614_transpose_x_0, transpose_y = var_614_transpose_y_0, x = var_613_cast_fp16, y = state_21_cast_fp16)[name = string("op_614_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_21_cast_fp16 = sub(x = var_611_cast_fp16, y = var_614_cast_fp16)[name = string("v_new_21_cast_fp16")];
+            tensor<int32, [4]> var_617_begin_0 = const()[name = string("op_617_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_617_end_0 = const()[name = string("op_617_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_617_end_mask_0 = const()[name = string("op_617_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_617_squeeze_mask_0 = const()[name = string("op_617_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_617_cast_fp16 = slice_by_index(begin = var_617_begin_0, end = var_617_end_0, end_mask = var_617_end_mask_0, squeeze_mask = var_617_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_617_cast_fp16")];
+            bool var_618_transpose_x_0 = const()[name = string("op_618_transpose_x_0"), val = bool(false)];
+            bool var_618_transpose_y_0 = const()[name = string("op_618_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_618_cast_fp16 = matmul(transpose_x = var_618_transpose_x_0, transpose_y = var_618_transpose_y_0, x = var_617_cast_fp16, y = state_21_cast_fp16)[name = string("op_618_cast_fp16")];
+            tensor<int32, [4]> var_620_begin_0 = const()[name = string("op_620_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_620_end_0 = const()[name = string("op_620_end_0"), val = tensor<int32, [4]>([32, 11, 64, 64])];
+            tensor<bool, [4]> var_620_end_mask_0 = const()[name = string("op_620_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_620_squeeze_mask_0 = const()[name = string("op_620_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_620_cast_fp16 = slice_by_index(begin = var_620_begin_0, end = var_620_end_0, end_mask = var_620_end_mask_0, squeeze_mask = var_620_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_620_cast_fp16")];
+            bool var_621_transpose_x_0 = const()[name = string("op_621_transpose_x_0"), val = bool(false)];
+            bool var_621_transpose_y_0 = const()[name = string("op_621_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_621_cast_fp16 = matmul(transpose_x = var_621_transpose_x_0, transpose_y = var_621_transpose_y_0, x = var_620_cast_fp16, y = v_new_21_cast_fp16)[name = string("op_621_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_622_cast_fp16 = add(x = var_618_cast_fp16, y = var_621_cast_fp16)[name = string("op_622_cast_fp16")];
+            tensor<int32, [4]> var_624_begin_0 = const()[name = string("op_624_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_624_end_0 = const()[name = string("op_624_end_0"), val = tensor<int32, [4]>([32, 11, 1, 1])];
+            tensor<bool, [4]> var_624_end_mask_0 = const()[name = string("op_624_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_624_squeeze_mask_0 = const()[name = string("op_624_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_624_cast_fp16 = slice_by_index(begin = var_624_begin_0, end = var_624_end_0, end_mask = var_624_end_mask_0, squeeze_mask = var_624_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_624_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_625_cast_fp16 = mul(x = state_21_cast_fp16, y = var_624_cast_fp16)[name = string("op_625_cast_fp16")];
+            tensor<int32, [4]> var_627_begin_0 = const()[name = string("op_627_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_627_end_0 = const()[name = string("op_627_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_627_end_mask_0 = const()[name = string("op_627_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_627_squeeze_mask_0 = const()[name = string("op_627_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_627_cast_fp16 = slice_by_index(begin = var_627_begin_0, end = var_627_end_0, end_mask = var_627_end_mask_0, squeeze_mask = var_627_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_627_cast_fp16")];
+            bool var_629_transpose_x_1 = const()[name = string("op_629_transpose_x_1"), val = bool(true)];
+            bool var_629_transpose_y_1 = const()[name = string("op_629_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_629_cast_fp16 = matmul(transpose_x = var_629_transpose_x_1, transpose_y = var_629_transpose_y_1, x = var_627_cast_fp16, y = v_new_21_cast_fp16)[name = string("op_629_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_23_cast_fp16 = add(x = var_625_cast_fp16, y = var_629_cast_fp16)[name = string("state_23_cast_fp16")];
+            tensor<int32, [4]> var_632_begin_0 = const()[name = string("op_632_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_632_end_0 = const()[name = string("op_632_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_632_end_mask_0 = const()[name = string("op_632_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_632_squeeze_mask_0 = const()[name = string("op_632_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_632_cast_fp16 = slice_by_index(begin = var_632_begin_0, end = var_632_end_0, end_mask = var_632_end_mask_0, squeeze_mask = var_632_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_632_cast_fp16")];
+            tensor<int32, [4]> var_634_begin_0 = const()[name = string("op_634_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_634_end_0 = const()[name = string("op_634_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_634_end_mask_0 = const()[name = string("op_634_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_634_squeeze_mask_0 = const()[name = string("op_634_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_634_cast_fp16 = slice_by_index(begin = var_634_begin_0, end = var_634_end_0, end_mask = var_634_end_mask_0, squeeze_mask = var_634_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_634_cast_fp16")];
+            bool var_635_transpose_x_0 = const()[name = string("op_635_transpose_x_0"), val = bool(false)];
+            bool var_635_transpose_y_0 = const()[name = string("op_635_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_635_cast_fp16 = matmul(transpose_x = var_635_transpose_x_0, transpose_y = var_635_transpose_y_0, x = var_634_cast_fp16, y = state_23_cast_fp16)[name = string("op_635_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_23_cast_fp16 = sub(x = var_632_cast_fp16, y = var_635_cast_fp16)[name = string("v_new_23_cast_fp16")];
+            tensor<int32, [4]> var_638_begin_0 = const()[name = string("op_638_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_638_end_0 = const()[name = string("op_638_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_638_end_mask_0 = const()[name = string("op_638_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_638_squeeze_mask_0 = const()[name = string("op_638_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_638_cast_fp16 = slice_by_index(begin = var_638_begin_0, end = var_638_end_0, end_mask = var_638_end_mask_0, squeeze_mask = var_638_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_638_cast_fp16")];
+            bool var_639_transpose_x_0 = const()[name = string("op_639_transpose_x_0"), val = bool(false)];
+            bool var_639_transpose_y_0 = const()[name = string("op_639_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_639_cast_fp16 = matmul(transpose_x = var_639_transpose_x_0, transpose_y = var_639_transpose_y_0, x = var_638_cast_fp16, y = state_23_cast_fp16)[name = string("op_639_cast_fp16")];
+            tensor<int32, [4]> var_641_begin_0 = const()[name = string("op_641_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_641_end_0 = const()[name = string("op_641_end_0"), val = tensor<int32, [4]>([32, 12, 64, 64])];
+            tensor<bool, [4]> var_641_end_mask_0 = const()[name = string("op_641_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_641_squeeze_mask_0 = const()[name = string("op_641_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_641_cast_fp16 = slice_by_index(begin = var_641_begin_0, end = var_641_end_0, end_mask = var_641_end_mask_0, squeeze_mask = var_641_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_641_cast_fp16")];
+            bool var_642_transpose_x_0 = const()[name = string("op_642_transpose_x_0"), val = bool(false)];
+            bool var_642_transpose_y_0 = const()[name = string("op_642_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_642_cast_fp16 = matmul(transpose_x = var_642_transpose_x_0, transpose_y = var_642_transpose_y_0, x = var_641_cast_fp16, y = v_new_23_cast_fp16)[name = string("op_642_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_643_cast_fp16 = add(x = var_639_cast_fp16, y = var_642_cast_fp16)[name = string("op_643_cast_fp16")];
+            tensor<int32, [4]> var_645_begin_0 = const()[name = string("op_645_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_645_end_0 = const()[name = string("op_645_end_0"), val = tensor<int32, [4]>([32, 12, 1, 1])];
+            tensor<bool, [4]> var_645_end_mask_0 = const()[name = string("op_645_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_645_squeeze_mask_0 = const()[name = string("op_645_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_645_cast_fp16 = slice_by_index(begin = var_645_begin_0, end = var_645_end_0, end_mask = var_645_end_mask_0, squeeze_mask = var_645_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_645_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_646_cast_fp16 = mul(x = state_23_cast_fp16, y = var_645_cast_fp16)[name = string("op_646_cast_fp16")];
+            tensor<int32, [4]> var_648_begin_0 = const()[name = string("op_648_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_648_end_0 = const()[name = string("op_648_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_648_end_mask_0 = const()[name = string("op_648_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_648_squeeze_mask_0 = const()[name = string("op_648_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_648_cast_fp16 = slice_by_index(begin = var_648_begin_0, end = var_648_end_0, end_mask = var_648_end_mask_0, squeeze_mask = var_648_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_648_cast_fp16")];
+            bool var_650_transpose_x_1 = const()[name = string("op_650_transpose_x_1"), val = bool(true)];
+            bool var_650_transpose_y_1 = const()[name = string("op_650_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_650_cast_fp16 = matmul(transpose_x = var_650_transpose_x_1, transpose_y = var_650_transpose_y_1, x = var_648_cast_fp16, y = v_new_23_cast_fp16)[name = string("op_650_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_25_cast_fp16 = add(x = var_646_cast_fp16, y = var_650_cast_fp16)[name = string("state_25_cast_fp16")];
+            tensor<int32, [4]> var_653_begin_0 = const()[name = string("op_653_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_653_end_0 = const()[name = string("op_653_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_653_end_mask_0 = const()[name = string("op_653_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_653_squeeze_mask_0 = const()[name = string("op_653_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_653_cast_fp16 = slice_by_index(begin = var_653_begin_0, end = var_653_end_0, end_mask = var_653_end_mask_0, squeeze_mask = var_653_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_653_cast_fp16")];
+            tensor<int32, [4]> var_655_begin_0 = const()[name = string("op_655_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_655_end_0 = const()[name = string("op_655_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_655_end_mask_0 = const()[name = string("op_655_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_655_squeeze_mask_0 = const()[name = string("op_655_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_655_cast_fp16 = slice_by_index(begin = var_655_begin_0, end = var_655_end_0, end_mask = var_655_end_mask_0, squeeze_mask = var_655_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_655_cast_fp16")];
+            bool var_656_transpose_x_0 = const()[name = string("op_656_transpose_x_0"), val = bool(false)];
+            bool var_656_transpose_y_0 = const()[name = string("op_656_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_656_cast_fp16 = matmul(transpose_x = var_656_transpose_x_0, transpose_y = var_656_transpose_y_0, x = var_655_cast_fp16, y = state_25_cast_fp16)[name = string("op_656_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_25_cast_fp16 = sub(x = var_653_cast_fp16, y = var_656_cast_fp16)[name = string("v_new_25_cast_fp16")];
+            tensor<int32, [4]> var_659_begin_0 = const()[name = string("op_659_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_659_end_0 = const()[name = string("op_659_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_659_end_mask_0 = const()[name = string("op_659_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_659_squeeze_mask_0 = const()[name = string("op_659_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_659_cast_fp16 = slice_by_index(begin = var_659_begin_0, end = var_659_end_0, end_mask = var_659_end_mask_0, squeeze_mask = var_659_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_659_cast_fp16")];
+            bool var_660_transpose_x_0 = const()[name = string("op_660_transpose_x_0"), val = bool(false)];
+            bool var_660_transpose_y_0 = const()[name = string("op_660_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_660_cast_fp16 = matmul(transpose_x = var_660_transpose_x_0, transpose_y = var_660_transpose_y_0, x = var_659_cast_fp16, y = state_25_cast_fp16)[name = string("op_660_cast_fp16")];
+            tensor<int32, [4]> var_662_begin_0 = const()[name = string("op_662_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_662_end_0 = const()[name = string("op_662_end_0"), val = tensor<int32, [4]>([32, 13, 64, 64])];
+            tensor<bool, [4]> var_662_end_mask_0 = const()[name = string("op_662_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_662_squeeze_mask_0 = const()[name = string("op_662_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_662_cast_fp16 = slice_by_index(begin = var_662_begin_0, end = var_662_end_0, end_mask = var_662_end_mask_0, squeeze_mask = var_662_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_662_cast_fp16")];
+            bool var_663_transpose_x_0 = const()[name = string("op_663_transpose_x_0"), val = bool(false)];
+            bool var_663_transpose_y_0 = const()[name = string("op_663_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_663_cast_fp16 = matmul(transpose_x = var_663_transpose_x_0, transpose_y = var_663_transpose_y_0, x = var_662_cast_fp16, y = v_new_25_cast_fp16)[name = string("op_663_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_664_cast_fp16 = add(x = var_660_cast_fp16, y = var_663_cast_fp16)[name = string("op_664_cast_fp16")];
+            tensor<int32, [4]> var_666_begin_0 = const()[name = string("op_666_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_666_end_0 = const()[name = string("op_666_end_0"), val = tensor<int32, [4]>([32, 13, 1, 1])];
+            tensor<bool, [4]> var_666_end_mask_0 = const()[name = string("op_666_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_666_squeeze_mask_0 = const()[name = string("op_666_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_666_cast_fp16 = slice_by_index(begin = var_666_begin_0, end = var_666_end_0, end_mask = var_666_end_mask_0, squeeze_mask = var_666_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_666_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_667_cast_fp16 = mul(x = state_25_cast_fp16, y = var_666_cast_fp16)[name = string("op_667_cast_fp16")];
+            tensor<int32, [4]> var_669_begin_0 = const()[name = string("op_669_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_669_end_0 = const()[name = string("op_669_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_669_end_mask_0 = const()[name = string("op_669_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_669_squeeze_mask_0 = const()[name = string("op_669_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_669_cast_fp16 = slice_by_index(begin = var_669_begin_0, end = var_669_end_0, end_mask = var_669_end_mask_0, squeeze_mask = var_669_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_669_cast_fp16")];
+            bool var_671_transpose_x_1 = const()[name = string("op_671_transpose_x_1"), val = bool(true)];
+            bool var_671_transpose_y_1 = const()[name = string("op_671_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_671_cast_fp16 = matmul(transpose_x = var_671_transpose_x_1, transpose_y = var_671_transpose_y_1, x = var_669_cast_fp16, y = v_new_25_cast_fp16)[name = string("op_671_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_27_cast_fp16 = add(x = var_667_cast_fp16, y = var_671_cast_fp16)[name = string("state_27_cast_fp16")];
+            tensor<int32, [4]> var_674_begin_0 = const()[name = string("op_674_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_674_end_0 = const()[name = string("op_674_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_674_end_mask_0 = const()[name = string("op_674_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_674_squeeze_mask_0 = const()[name = string("op_674_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_674_cast_fp16 = slice_by_index(begin = var_674_begin_0, end = var_674_end_0, end_mask = var_674_end_mask_0, squeeze_mask = var_674_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_674_cast_fp16")];
+            tensor<int32, [4]> var_676_begin_0 = const()[name = string("op_676_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_676_end_0 = const()[name = string("op_676_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_676_end_mask_0 = const()[name = string("op_676_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_676_squeeze_mask_0 = const()[name = string("op_676_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_676_cast_fp16 = slice_by_index(begin = var_676_begin_0, end = var_676_end_0, end_mask = var_676_end_mask_0, squeeze_mask = var_676_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_676_cast_fp16")];
+            bool var_677_transpose_x_0 = const()[name = string("op_677_transpose_x_0"), val = bool(false)];
+            bool var_677_transpose_y_0 = const()[name = string("op_677_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_677_cast_fp16 = matmul(transpose_x = var_677_transpose_x_0, transpose_y = var_677_transpose_y_0, x = var_676_cast_fp16, y = state_27_cast_fp16)[name = string("op_677_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_27_cast_fp16 = sub(x = var_674_cast_fp16, y = var_677_cast_fp16)[name = string("v_new_27_cast_fp16")];
+            tensor<int32, [4]> var_680_begin_0 = const()[name = string("op_680_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_680_end_0 = const()[name = string("op_680_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_680_end_mask_0 = const()[name = string("op_680_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_680_squeeze_mask_0 = const()[name = string("op_680_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_680_cast_fp16 = slice_by_index(begin = var_680_begin_0, end = var_680_end_0, end_mask = var_680_end_mask_0, squeeze_mask = var_680_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_680_cast_fp16")];
+            bool var_681_transpose_x_0 = const()[name = string("op_681_transpose_x_0"), val = bool(false)];
+            bool var_681_transpose_y_0 = const()[name = string("op_681_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_681_cast_fp16 = matmul(transpose_x = var_681_transpose_x_0, transpose_y = var_681_transpose_y_0, x = var_680_cast_fp16, y = state_27_cast_fp16)[name = string("op_681_cast_fp16")];
+            tensor<int32, [4]> var_683_begin_0 = const()[name = string("op_683_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_683_end_0 = const()[name = string("op_683_end_0"), val = tensor<int32, [4]>([32, 14, 64, 64])];
+            tensor<bool, [4]> var_683_end_mask_0 = const()[name = string("op_683_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_683_squeeze_mask_0 = const()[name = string("op_683_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_683_cast_fp16 = slice_by_index(begin = var_683_begin_0, end = var_683_end_0, end_mask = var_683_end_mask_0, squeeze_mask = var_683_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_683_cast_fp16")];
+            bool var_684_transpose_x_0 = const()[name = string("op_684_transpose_x_0"), val = bool(false)];
+            bool var_684_transpose_y_0 = const()[name = string("op_684_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_684_cast_fp16 = matmul(transpose_x = var_684_transpose_x_0, transpose_y = var_684_transpose_y_0, x = var_683_cast_fp16, y = v_new_27_cast_fp16)[name = string("op_684_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_685_cast_fp16 = add(x = var_681_cast_fp16, y = var_684_cast_fp16)[name = string("op_685_cast_fp16")];
+            tensor<int32, [4]> var_687_begin_0 = const()[name = string("op_687_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_687_end_0 = const()[name = string("op_687_end_0"), val = tensor<int32, [4]>([32, 14, 1, 1])];
+            tensor<bool, [4]> var_687_end_mask_0 = const()[name = string("op_687_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_687_squeeze_mask_0 = const()[name = string("op_687_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_687_cast_fp16 = slice_by_index(begin = var_687_begin_0, end = var_687_end_0, end_mask = var_687_end_mask_0, squeeze_mask = var_687_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_687_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_688_cast_fp16 = mul(x = state_27_cast_fp16, y = var_687_cast_fp16)[name = string("op_688_cast_fp16")];
+            tensor<int32, [4]> var_690_begin_0 = const()[name = string("op_690_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_690_end_0 = const()[name = string("op_690_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_690_end_mask_0 = const()[name = string("op_690_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_690_squeeze_mask_0 = const()[name = string("op_690_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_690_cast_fp16 = slice_by_index(begin = var_690_begin_0, end = var_690_end_0, end_mask = var_690_end_mask_0, squeeze_mask = var_690_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_690_cast_fp16")];
+            bool var_692_transpose_x_1 = const()[name = string("op_692_transpose_x_1"), val = bool(true)];
+            bool var_692_transpose_y_1 = const()[name = string("op_692_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_692_cast_fp16 = matmul(transpose_x = var_692_transpose_x_1, transpose_y = var_692_transpose_y_1, x = var_690_cast_fp16, y = v_new_27_cast_fp16)[name = string("op_692_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_29_cast_fp16 = add(x = var_688_cast_fp16, y = var_692_cast_fp16)[name = string("state_29_cast_fp16")];
+            tensor<int32, [4]> var_695_begin_0 = const()[name = string("op_695_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_695_end_0 = const()[name = string("op_695_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_695_end_mask_0 = const()[name = string("op_695_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_695_squeeze_mask_0 = const()[name = string("op_695_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_695_cast_fp16 = slice_by_index(begin = var_695_begin_0, end = var_695_end_0, end_mask = var_695_end_mask_0, squeeze_mask = var_695_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_695_cast_fp16")];
+            tensor<int32, [4]> var_697_begin_0 = const()[name = string("op_697_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_697_end_0 = const()[name = string("op_697_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_697_end_mask_0 = const()[name = string("op_697_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_697_squeeze_mask_0 = const()[name = string("op_697_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_697_cast_fp16 = slice_by_index(begin = var_697_begin_0, end = var_697_end_0, end_mask = var_697_end_mask_0, squeeze_mask = var_697_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_697_cast_fp16")];
+            bool var_698_transpose_x_0 = const()[name = string("op_698_transpose_x_0"), val = bool(false)];
+            bool var_698_transpose_y_0 = const()[name = string("op_698_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_698_cast_fp16 = matmul(transpose_x = var_698_transpose_x_0, transpose_y = var_698_transpose_y_0, x = var_697_cast_fp16, y = state_29_cast_fp16)[name = string("op_698_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_29_cast_fp16 = sub(x = var_695_cast_fp16, y = var_698_cast_fp16)[name = string("v_new_29_cast_fp16")];
+            tensor<int32, [4]> var_701_begin_0 = const()[name = string("op_701_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_701_end_0 = const()[name = string("op_701_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_701_end_mask_0 = const()[name = string("op_701_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_701_squeeze_mask_0 = const()[name = string("op_701_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_701_cast_fp16 = slice_by_index(begin = var_701_begin_0, end = var_701_end_0, end_mask = var_701_end_mask_0, squeeze_mask = var_701_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_701_cast_fp16")];
+            bool var_702_transpose_x_0 = const()[name = string("op_702_transpose_x_0"), val = bool(false)];
+            bool var_702_transpose_y_0 = const()[name = string("op_702_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_702_cast_fp16 = matmul(transpose_x = var_702_transpose_x_0, transpose_y = var_702_transpose_y_0, x = var_701_cast_fp16, y = state_29_cast_fp16)[name = string("op_702_cast_fp16")];
+            tensor<int32, [4]> var_704_begin_0 = const()[name = string("op_704_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_704_end_0 = const()[name = string("op_704_end_0"), val = tensor<int32, [4]>([32, 15, 64, 64])];
+            tensor<bool, [4]> var_704_end_mask_0 = const()[name = string("op_704_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_704_squeeze_mask_0 = const()[name = string("op_704_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_704_cast_fp16 = slice_by_index(begin = var_704_begin_0, end = var_704_end_0, end_mask = var_704_end_mask_0, squeeze_mask = var_704_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_704_cast_fp16")];
+            bool var_705_transpose_x_0 = const()[name = string("op_705_transpose_x_0"), val = bool(false)];
+            bool var_705_transpose_y_0 = const()[name = string("op_705_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_705_cast_fp16 = matmul(transpose_x = var_705_transpose_x_0, transpose_y = var_705_transpose_y_0, x = var_704_cast_fp16, y = v_new_29_cast_fp16)[name = string("op_705_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_706_cast_fp16 = add(x = var_702_cast_fp16, y = var_705_cast_fp16)[name = string("op_706_cast_fp16")];
+            tensor<int32, [4]> var_708_begin_0 = const()[name = string("op_708_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_708_end_0 = const()[name = string("op_708_end_0"), val = tensor<int32, [4]>([32, 15, 1, 1])];
+            tensor<bool, [4]> var_708_end_mask_0 = const()[name = string("op_708_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_708_squeeze_mask_0 = const()[name = string("op_708_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_708_cast_fp16 = slice_by_index(begin = var_708_begin_0, end = var_708_end_0, end_mask = var_708_end_mask_0, squeeze_mask = var_708_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_708_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_709_cast_fp16 = mul(x = state_29_cast_fp16, y = var_708_cast_fp16)[name = string("op_709_cast_fp16")];
+            tensor<int32, [4]> var_711_begin_0 = const()[name = string("op_711_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_711_end_0 = const()[name = string("op_711_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_711_end_mask_0 = const()[name = string("op_711_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_711_squeeze_mask_0 = const()[name = string("op_711_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_711_cast_fp16 = slice_by_index(begin = var_711_begin_0, end = var_711_end_0, end_mask = var_711_end_mask_0, squeeze_mask = var_711_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_711_cast_fp16")];
+            bool var_713_transpose_x_1 = const()[name = string("op_713_transpose_x_1"), val = bool(true)];
+            bool var_713_transpose_y_1 = const()[name = string("op_713_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_713_cast_fp16 = matmul(transpose_x = var_713_transpose_x_1, transpose_y = var_713_transpose_y_1, x = var_711_cast_fp16, y = v_new_29_cast_fp16)[name = string("op_713_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_31_cast_fp16 = add(x = var_709_cast_fp16, y = var_713_cast_fp16)[name = string("state_31_cast_fp16")];
+            tensor<int32, [4]> var_716_begin_0 = const()[name = string("op_716_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_716_end_0 = const()[name = string("op_716_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_716_end_mask_0 = const()[name = string("op_716_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_716_squeeze_mask_0 = const()[name = string("op_716_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_716_cast_fp16 = slice_by_index(begin = var_716_begin_0, end = var_716_end_0, end_mask = var_716_end_mask_0, squeeze_mask = var_716_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_716_cast_fp16")];
+            tensor<int32, [4]> var_718_begin_0 = const()[name = string("op_718_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_718_end_0 = const()[name = string("op_718_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_718_end_mask_0 = const()[name = string("op_718_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_718_squeeze_mask_0 = const()[name = string("op_718_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_718_cast_fp16 = slice_by_index(begin = var_718_begin_0, end = var_718_end_0, end_mask = var_718_end_mask_0, squeeze_mask = var_718_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_718_cast_fp16")];
+            bool var_719_transpose_x_0 = const()[name = string("op_719_transpose_x_0"), val = bool(false)];
+            bool var_719_transpose_y_0 = const()[name = string("op_719_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_719_cast_fp16 = matmul(transpose_x = var_719_transpose_x_0, transpose_y = var_719_transpose_y_0, x = var_718_cast_fp16, y = state_31_cast_fp16)[name = string("op_719_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_31_cast_fp16 = sub(x = var_716_cast_fp16, y = var_719_cast_fp16)[name = string("v_new_31_cast_fp16")];
+            tensor<int32, [4]> var_722_begin_0 = const()[name = string("op_722_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_722_end_0 = const()[name = string("op_722_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_722_end_mask_0 = const()[name = string("op_722_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_722_squeeze_mask_0 = const()[name = string("op_722_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_722_cast_fp16 = slice_by_index(begin = var_722_begin_0, end = var_722_end_0, end_mask = var_722_end_mask_0, squeeze_mask = var_722_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_722_cast_fp16")];
+            bool var_723_transpose_x_0 = const()[name = string("op_723_transpose_x_0"), val = bool(false)];
+            bool var_723_transpose_y_0 = const()[name = string("op_723_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_723_cast_fp16 = matmul(transpose_x = var_723_transpose_x_0, transpose_y = var_723_transpose_y_0, x = var_722_cast_fp16, y = state_31_cast_fp16)[name = string("op_723_cast_fp16")];
+            tensor<int32, [4]> var_725_begin_0 = const()[name = string("op_725_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_725_end_0 = const()[name = string("op_725_end_0"), val = tensor<int32, [4]>([32, 16, 64, 64])];
+            tensor<bool, [4]> var_725_end_mask_0 = const()[name = string("op_725_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_725_squeeze_mask_0 = const()[name = string("op_725_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_725_cast_fp16 = slice_by_index(begin = var_725_begin_0, end = var_725_end_0, end_mask = var_725_end_mask_0, squeeze_mask = var_725_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_725_cast_fp16")];
+            bool var_726_transpose_x_0 = const()[name = string("op_726_transpose_x_0"), val = bool(false)];
+            bool var_726_transpose_y_0 = const()[name = string("op_726_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_726_cast_fp16 = matmul(transpose_x = var_726_transpose_x_0, transpose_y = var_726_transpose_y_0, x = var_725_cast_fp16, y = v_new_31_cast_fp16)[name = string("op_726_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_727_cast_fp16 = add(x = var_723_cast_fp16, y = var_726_cast_fp16)[name = string("op_727_cast_fp16")];
+            int32 var_729_axis_0 = const()[name = string("op_729_axis_0"), val = int32(1)];
+            tensor<fp16, [32, 16, 64, 128]> var_729_cast_fp16 = stack(axis = var_729_axis_0, values = (var_412_cast_fp16, var_433_cast_fp16, var_454_cast_fp16, var_475_cast_fp16, var_496_cast_fp16, var_517_cast_fp16, var_538_cast_fp16, var_559_cast_fp16, var_580_cast_fp16, var_601_cast_fp16, var_622_cast_fp16, var_643_cast_fp16, var_664_cast_fp16, var_685_cast_fp16, var_706_cast_fp16, var_727_cast_fp16))[name = string("op_729_cast_fp16")];
+            tensor<int32, [3]> var_730 = const()[name = string("op_730"), val = tensor<int32, [3]>([32, 1024, 128])];
+            tensor<fp16, [32, 1024, 128]> var_731_cast_fp16 = reshape(shape = var_730, x = var_729_cast_fp16)[name = string("op_731_cast_fp16")];
+            tensor<int32, [3]> var_732 = const()[name = string("op_732"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> input_7_begin_0 = const()[name = string("input_7_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> input_7_end_0 = const()[name = string("input_7_end_0"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<bool, [4]> input_7_end_mask_0 = const()[name = string("input_7_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> input_7_squeeze_mask_0 = const()[name = string("input_7_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [1024, 32, 128]> input_7_cast_fp16 = slice_by_index(begin = input_7_begin_0, end = input_7_end_0, end_mask = input_7_end_mask_0, squeeze_mask = input_7_squeeze_mask_0, x = z_1_cast_fp16)[name = string("input_7_cast_fp16")];
+            fp16 var_48_promoted_1_to_fp16 = const()[name = string("op_48_promoted_1_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1024, 32, 128]> x_17_cast_fp16 = transpose(perm = var_732, x = var_731_cast_fp16)[name = string("transpose_48")];
+            tensor<fp16, [1024, 32, 128]> var_736_cast_fp16 = pow(x = x_17_cast_fp16, y = var_48_promoted_1_to_fp16)[name = string("op_736_cast_fp16")];
+            tensor<int32, [1]> var_738_axes_0 = const()[name = string("op_738_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_738_keep_dims_0 = const()[name = string("op_738_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1024, 32, 1]> var_738_cast_fp16 = reduce_mean(axes = var_738_axes_0, keep_dims = var_738_keep_dims_0, x = var_736_cast_fp16)[name = string("op_738_cast_fp16")];
+            fp16 var_739_to_fp16 = const()[name = string("op_739_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1024, 32, 1]> var_740_cast_fp16 = add(x = var_738_cast_fp16, y = var_739_to_fp16)[name = string("op_740_cast_fp16")];
+            fp32 var_741_epsilon_0 = const()[name = string("op_741_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1024, 32, 1]> var_741_cast_fp16 = rsqrt(epsilon = var_741_epsilon_0, x = var_740_cast_fp16)[name = string("op_741_cast_fp16")];
+            tensor<fp16, [1024, 32, 128]> x_19_cast_fp16 = mul(x = x_17_cast_fp16, y = var_741_cast_fp16)[name = string("x_19_cast_fp16")];
+            tensor<fp16, [128]> layers_0_linear_attn_norm_weight_to_fp16 = const()[name = string("layers_0_linear_attn_norm_weight_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(685501888)))];
+            tensor<fp16, [1024, 32, 128]> var_743_cast_fp16 = mul(x = layers_0_linear_attn_norm_weight_to_fp16, y = x_19_cast_fp16)[name = string("op_743_cast_fp16")];
+            tensor<fp16, [1024, 32, 128]> var_744_cast_fp16 = silu(x = input_7_cast_fp16)[name = string("op_744_cast_fp16")];
+            tensor<fp16, [1024, 32, 128]> core_1_cast_fp16 = mul(x = var_743_cast_fp16, y = var_744_cast_fp16)[name = string("core_1_cast_fp16")];
+            tensor<int32, [3]> var_746 = const()[name = string("op_746"), val = tensor<int32, [3]>([1, 1024, 4096])];
+            tensor<fp16, [1, 1024, 4096]> input_9_cast_fp16 = reshape(shape = var_746, x = core_1_cast_fp16)[name = string("input_9_cast_fp16")];
+            tensor<fp16, [2560]> linear_4_bias_0_to_fp16 = const()[name = string("linear_4_bias_0_to_fp16"), val = tensor<fp16, [2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(685502208)))];
+            tensor<fp16, [1, 1024, 2560]> linear_4_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers_0_linear_attn_out_proj_weight_cast_fp16, x = input_9_cast_fp16)[name = string("linear_4_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> x_21_cast_fp16 = add(x = hidden, y = linear_4_cast_fp16)[name = string("x_21_cast_fp16")];
+            fp16 var_48_promoted_2_to_fp16 = const()[name = string("op_48_promoted_2_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 1024, 2560]> var_752_cast_fp16 = pow(x = x_21_cast_fp16, y = var_48_promoted_2_to_fp16)[name = string("op_752_cast_fp16")];
+            tensor<int32, [1]> var_754_axes_0 = const()[name = string("op_754_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_754_keep_dims_0 = const()[name = string("op_754_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 1024, 1]> var_754_cast_fp16 = reduce_mean(axes = var_754_axes_0, keep_dims = var_754_keep_dims_0, x = var_752_cast_fp16)[name = string("op_754_cast_fp16")];
+            fp16 var_755_to_fp16 = const()[name = string("op_755_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 1024, 1]> var_756_cast_fp16 = add(x = var_754_cast_fp16, y = var_755_to_fp16)[name = string("op_756_cast_fp16")];
+            fp32 var_757_epsilon_0 = const()[name = string("op_757_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 1024, 1]> var_757_cast_fp16 = rsqrt(epsilon = var_757_epsilon_0, x = var_756_cast_fp16)[name = string("op_757_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> x_23_cast_fp16 = mul(x = x_21_cast_fp16, y = var_757_cast_fp16)[name = string("x_23_cast_fp16")];
+            tensor<fp16, [2560]> var_760_to_fp16 = const()[name = string("op_760_to_fp16"), val = tensor<fp16, [2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(685507392)))];
+            tensor<fp16, [1, 1024, 2560]> input_11_cast_fp16 = mul(x = x_23_cast_fp16, y = var_760_to_fp16)[name = string("input_11_cast_fp16")];
+            tensor<fp16, [9216]> linear_5_bias_0_to_fp16 = const()[name = string("linear_5_bias_0_to_fp16"), val = tensor<fp16, [9216]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(685512576)))];
+            tensor<fp16, [1, 1024, 9216]> linear_5_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = layers_0_mlp_gate_proj_weight_cast_fp16, x = input_11_cast_fp16)[name = string("linear_5_cast_fp16")];
+            tensor<fp16, [1, 1024, 9216]> var_767_cast_fp16 = silu(x = linear_5_cast_fp16)[name = string("op_767_cast_fp16")];
+            tensor<fp16, [1, 1024, 9216]> linear_6_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = layers_0_mlp_up_proj_weight_cast_fp16, x = input_11_cast_fp16)[name = string("linear_6_cast_fp16")];
+            tensor<fp16, [1, 1024, 9216]> input_15_cast_fp16 = mul(x = var_767_cast_fp16, y = linear_6_cast_fp16)[name = string("input_15_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> linear_7_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers_0_mlp_down_proj_weight_cast_fp16, x = input_15_cast_fp16)[name = string("linear_7_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> x_25_cast_fp16 = add(x = x_21_cast_fp16, y = linear_7_cast_fp16)[name = string("x_25_cast_fp16")];
+            int32 var_787 = const()[name = string("op_787"), val = int32(-2)];
+            int32 var_802 = const()[name = string("op_802"), val = int32(-1)];
+            fp16 var_801_promoted_to_fp16 = const()[name = string("op_801_promoted_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 1024, 2560]> var_811_cast_fp16 = pow(x = x_25_cast_fp16, y = var_801_promoted_to_fp16)[name = string("op_811_cast_fp16")];
+            tensor<int32, [1]> var_813_axes_0 = const()[name = string("op_813_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_813_keep_dims_0 = const()[name = string("op_813_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 1024, 1]> var_813_cast_fp16 = reduce_mean(axes = var_813_axes_0, keep_dims = var_813_keep_dims_0, x = var_811_cast_fp16)[name = string("op_813_cast_fp16")];
+            fp16 var_814_to_fp16 = const()[name = string("op_814_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 1024, 1]> var_815_cast_fp16 = add(x = var_813_cast_fp16, y = var_814_to_fp16)[name = string("op_815_cast_fp16")];
+            fp32 var_816_epsilon_0 = const()[name = string("op_816_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 1024, 1]> var_816_cast_fp16 = rsqrt(epsilon = var_816_epsilon_0, x = var_815_cast_fp16)[name = string("op_816_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> x_27_cast_fp16 = mul(x = x_25_cast_fp16, y = var_816_cast_fp16)[name = string("x_27_cast_fp16")];
+            tensor<fp16, [2560]> var_819_to_fp16 = const()[name = string("op_819_to_fp16"), val = tensor<fp16, [2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(685531072)))];
+            tensor<fp16, [1, 1024, 2560]> x_29_cast_fp16 = mul(x = x_27_cast_fp16, y = var_819_to_fp16)[name = string("x_29_cast_fp16")];
+            tensor<fp16, [1, 1024, 8192]> linear_8_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = layers_1_linear_attn_in_proj_qkv_weight_cast_fp16, x = x_29_cast_fp16)[name = string("linear_8_cast_fp16")];
+            tensor<int32, [3]> input_17_perm_0 = const()[name = string("input_17_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
+            tensor<int32, [6]> input_19_pad_0 = const()[name = string("input_19_pad_0"), val = tensor<int32, [6]>([0, 0, 0, 0, 3, 0])];
+            string input_19_mode_0 = const()[name = string("input_19_mode_0"), val = string("constant")];
+            fp16 const_13_to_fp16 = const()[name = string("const_13_to_fp16"), val = fp16(0x0p+0)];
+            tensor<fp16, [1, 8192, 1024]> input_17_cast_fp16 = transpose(perm = input_17_perm_0, x = linear_8_cast_fp16)[name = string("transpose_47")];
+            tensor<fp16, [1, 8192, 1027]> input_19_cast_fp16 = pad(constant_val = const_13_to_fp16, mode = input_19_mode_0, pad = input_19_pad_0, x = input_17_cast_fp16)[name = string("input_19_cast_fp16")];
+            string input_21_pad_type_0 = const()[name = string("input_21_pad_type_0"), val = string("valid")];
+            int32 input_21_groups_0 = const()[name = string("input_21_groups_0"), val = int32(8192)];
+            tensor<int32, [1]> input_21_strides_0 = const()[name = string("input_21_strides_0"), val = tensor<int32, [1]>([1])];
+            tensor<int32, [2]> input_21_pad_0 = const()[name = string("input_21_pad_0"), val = tensor<int32, [2]>([0, 0])];
+            tensor<int32, [1]> input_21_dilations_0 = const()[name = string("input_21_dilations_0"), val = tensor<int32, [1]>([1])];
+            tensor<fp16, [8192, 1, 4]> layers_1_linear_attn_conv1d_weight_to_fp16 = const()[name = string("layers_1_linear_attn_conv1d_weight_to_fp16"), val = tensor<fp16, [8192, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(685536256)))];
+            tensor<fp16, [1, 8192, 1024]> input_21_cast_fp16 = conv(dilations = input_21_dilations_0, groups = input_21_groups_0, pad = input_21_pad_0, pad_type = input_21_pad_type_0, strides = input_21_strides_0, weight = layers_1_linear_attn_conv1d_weight_to_fp16, x = input_19_cast_fp16)[name = string("input_21_cast_fp16")];
+            tensor<fp16, [1, 8192, 1024]> var_854_cast_fp16 = silu(x = input_21_cast_fp16)[name = string("op_854_cast_fp16")];
+            tensor<int32, [3]> qkv_3_perm_0 = const()[name = string("qkv_3_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
+            tensor<int32, [3]> var_856_begin_0 = const()[name = string("op_856_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_856_end_0 = const()[name = string("op_856_end_0"), val = tensor<int32, [3]>([1, 1024, 2048])];
+            tensor<bool, [3]> var_856_end_mask_0 = const()[name = string("op_856_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
+            tensor<fp16, [1, 1024, 8192]> qkv_3_cast_fp16 = transpose(perm = qkv_3_perm_0, x = var_854_cast_fp16)[name = string("transpose_46")];
+            tensor<fp16, [1, 1024, 2048]> var_856_cast_fp16 = slice_by_index(begin = var_856_begin_0, end = var_856_end_0, end_mask = var_856_end_mask_0, x = qkv_3_cast_fp16)[name = string("op_856_cast_fp16")];
+            tensor<int32, [4]> var_857 = const()[name = string("op_857"), val = tensor<int32, [4]>([1, 1024, 16, 128])];
+            tensor<fp16, [1, 1024, 16, 128]> q_11_cast_fp16 = reshape(shape = var_857, x = var_856_cast_fp16)[name = string("q_11_cast_fp16")];
+            tensor<int32, [3]> var_859_begin_0 = const()[name = string("op_859_begin_0"), val = tensor<int32, [3]>([0, 0, 2048])];
+            tensor<int32, [3]> var_859_end_0 = const()[name = string("op_859_end_0"), val = tensor<int32, [3]>([1, 1024, 4096])];
+            tensor<bool, [3]> var_859_end_mask_0 = const()[name = string("op_859_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
+            tensor<fp16, [1, 1024, 2048]> var_859_cast_fp16 = slice_by_index(begin = var_859_begin_0, end = var_859_end_0, end_mask = var_859_end_mask_0, x = qkv_3_cast_fp16)[name = string("op_859_cast_fp16")];
+            tensor<int32, [4]> var_860 = const()[name = string("op_860"), val = tensor<int32, [4]>([1, 1024, 16, 128])];
+            tensor<fp16, [1, 1024, 16, 128]> k_9_cast_fp16 = reshape(shape = var_860, x = var_859_cast_fp16)[name = string("k_9_cast_fp16")];
+            tensor<int32, [3]> var_862_begin_0 = const()[name = string("op_862_begin_0"), val = tensor<int32, [3]>([0, 0, 4096])];
+            tensor<int32, [3]> var_862_end_0 = const()[name = string("op_862_end_0"), val = tensor<int32, [3]>([1, 1024, 8192])];
+            tensor<bool, [3]> var_862_end_mask_0 = const()[name = string("op_862_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
+            tensor<fp16, [1, 1024, 4096]> var_862_cast_fp16 = slice_by_index(begin = var_862_begin_0, end = var_862_end_0, end_mask = var_862_end_mask_0, x = qkv_3_cast_fp16)[name = string("op_862_cast_fp16")];
+            tensor<int32, [4]> var_863 = const()[name = string("op_863"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<fp16, [1, 1024, 32, 128]> v_5_cast_fp16 = reshape(shape = var_863, x = var_862_cast_fp16)[name = string("v_5_cast_fp16")];
+            tensor<fp16, [1, 1024, 4096]> linear_9_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = layers_1_linear_attn_in_proj_z_weight_cast_fp16, x = x_29_cast_fp16)[name = string("linear_9_cast_fp16")];
+            tensor<int32, [4]> var_867 = const()[name = string("op_867"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<fp16, [1, 1024, 32, 128]> z_3_cast_fp16 = reshape(shape = var_867, x = linear_9_cast_fp16)[name = string("z_3_cast_fp16")];
+            tensor<fp16, [32]> linear_10_bias_0_to_fp16 = const()[name = string("linear_10_bias_0_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(685601856)))];
+            tensor<fp16, [1, 1024, 32]> linear_10_cast_fp16 = linear(bias = linear_10_bias_0_to_fp16, weight = layers_1_linear_attn_in_proj_b_weight_cast_fp16, x = x_29_cast_fp16)[name = string("linear_10_cast_fp16")];
+            tensor<fp16, [1, 1024, 32]> beta_5_cast_fp16 = sigmoid(x = linear_10_cast_fp16)[name = string("beta_5_cast_fp16")];
+            tensor<fp16, [32]> linear_11_bias_0_to_fp16 = const()[name = string("linear_11_bias_0_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(685601984)))];
+            tensor<fp16, [1, 1024, 32]> linear_11_cast_fp16 = linear(bias = linear_11_bias_0_to_fp16, weight = layers_1_linear_attn_in_proj_a_weight_cast_fp16, x = x_29_cast_fp16)[name = string("linear_11_cast_fp16")];
+            tensor<fp16, [32]> layers_1_linear_attn_dt_bias_to_fp16 = const()[name = string("layers_1_linear_attn_dt_bias_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(685602112)))];
+            tensor<fp16, [1, 1024, 32]> var_876_cast_fp16 = add(x = linear_11_cast_fp16, y = layers_1_linear_attn_dt_bias_to_fp16)[name = string("op_876_cast_fp16")];
+            tensor<fp16, [1, 1024, 32]> var_877_cast_fp16 = softplus(x = var_876_cast_fp16)[name = string("op_877_cast_fp16")];
+            tensor<fp16, [32]> var_873_to_fp16 = const()[name = string("op_873_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(685602240)))];
+            tensor<fp16, [1, 1024, 32]> g_5_cast_fp16 = mul(x = var_873_to_fp16, y = var_877_cast_fp16)[name = string("g_5_cast_fp16")];
+            tensor<fp16, [1, 1024, 16, 128]> var_879_cast_fp16 = mul(x = q_11_cast_fp16, y = q_11_cast_fp16)[name = string("op_879_cast_fp16")];
+            tensor<int32, [1]> var_881_axes_0 = const()[name = string("op_881_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_881_keep_dims_0 = const()[name = string("op_881_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 1024, 16, 1]> var_881_cast_fp16 = reduce_sum(axes = var_881_axes_0, keep_dims = var_881_keep_dims_0, x = var_879_cast_fp16)[name = string("op_881_cast_fp16")];
+            fp16 var_882_to_fp16 = const()[name = string("op_882_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 1024, 16, 1]> var_883_cast_fp16 = add(x = var_881_cast_fp16, y = var_882_to_fp16)[name = string("op_883_cast_fp16")];
+            fp32 var_884_epsilon_0 = const()[name = string("op_884_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 1024, 16, 1]> var_884_cast_fp16 = rsqrt(epsilon = var_884_epsilon_0, x = var_883_cast_fp16)[name = string("op_884_cast_fp16")];
+            tensor<fp16, [1, 1024, 16, 128]> q_13_cast_fp16 = mul(x = q_11_cast_fp16, y = var_884_cast_fp16)[name = string("q_13_cast_fp16")];
+            tensor<fp16, [1, 1024, 16, 128]> var_886_cast_fp16 = mul(x = k_9_cast_fp16, y = k_9_cast_fp16)[name = string("op_886_cast_fp16")];
+            tensor<int32, [1]> var_888_axes_0 = const()[name = string("op_888_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_888_keep_dims_0 = const()[name = string("op_888_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 1024, 16, 1]> var_888_cast_fp16 = reduce_sum(axes = var_888_axes_0, keep_dims = var_888_keep_dims_0, x = var_886_cast_fp16)[name = string("op_888_cast_fp16")];
+            fp16 var_889_to_fp16 = const()[name = string("op_889_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 1024, 16, 1]> var_890_cast_fp16 = add(x = var_888_cast_fp16, y = var_889_to_fp16)[name = string("op_890_cast_fp16")];
+            fp32 var_891_epsilon_0 = const()[name = string("op_891_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 1024, 16, 1]> var_891_cast_fp16 = rsqrt(epsilon = var_891_epsilon_0, x = var_890_cast_fp16)[name = string("op_891_cast_fp16")];
+            tensor<fp16, [1, 1024, 16, 128]> k_11_cast_fp16 = mul(x = k_9_cast_fp16, y = var_891_cast_fp16)[name = string("k_11_cast_fp16")];
+            tensor<int32, [1]> var_896_axes_0 = const()[name = string("op_896_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [1, 1024, 16, 1, 128]> var_896_cast_fp16 = expand_dims(axes = var_896_axes_0, x = q_13_cast_fp16)[name = string("op_896_cast_fp16")];
+            tensor<int32, [5]> var_898_reps_0 = const()[name = string("op_898_reps_0"), val = tensor<int32, [5]>([1, 1, 1, 2, 1])];
+            tensor<fp16, [1, 1024, 16, 2, 128]> var_898_cast_fp16 = tile(reps = var_898_reps_0, x = var_896_cast_fp16)[name = string("op_898_cast_fp16")];
+            tensor<int32, [4]> var_899 = const()[name = string("op_899"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<fp16, [1, 1024, 32, 128]> q_15_cast_fp16 = reshape(shape = var_899, x = var_898_cast_fp16)[name = string("q_15_cast_fp16")];
+            tensor<int32, [1]> var_904_axes_0 = const()[name = string("op_904_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [1, 1024, 16, 1, 128]> var_904_cast_fp16 = expand_dims(axes = var_904_axes_0, x = k_11_cast_fp16)[name = string("op_904_cast_fp16")];
+            tensor<int32, [5]> var_906_reps_0 = const()[name = string("op_906_reps_0"), val = tensor<int32, [5]>([1, 1, 1, 2, 1])];
+            tensor<fp16, [1, 1024, 16, 2, 128]> var_906_cast_fp16 = tile(reps = var_906_reps_0, x = var_904_cast_fp16)[name = string("op_906_cast_fp16")];
+            tensor<int32, [4]> var_907 = const()[name = string("op_907"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<fp16, [1, 1024, 32, 128]> k_13_cast_fp16 = reshape(shape = var_907, x = var_906_cast_fp16)[name = string("k_13_cast_fp16")];
+            fp16 var_909_to_fp16 = const()[name = string("op_909_to_fp16"), val = fp16(0x1.6ap-4)];
+            tensor<fp16, [1, 1024, 32, 128]> q_17_cast_fp16 = mul(x = q_15_cast_fp16, y = var_909_to_fp16)[name = string("q_17_cast_fp16")];
+            tensor<int32, [4]> var_911_begin_0 = const()[name = string("op_911_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_911_end_0 = const()[name = string("op_911_end_0"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<bool, [4]> var_911_end_mask_0 = const()[name = string("op_911_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_911_squeeze_mask_0 = const()[name = string("op_911_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [1024, 32, 128]> var_911_cast_fp16 = slice_by_index(begin = var_911_begin_0, end = var_911_end_0, end_mask = var_911_end_mask_0, squeeze_mask = var_911_squeeze_mask_0, x = q_17_cast_fp16)[name = string("op_911_cast_fp16")];
+            tensor<int32, [3]> var_912 = const()[name = string("op_912"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_914 = const()[name = string("op_914"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<fp16, [32, 1024, 128]> var_913_cast_fp16 = transpose(perm = var_912, x = var_911_cast_fp16)[name = string("transpose_45")];
+            tensor<fp16, [32, 16, 64, 128]> q_19_cast_fp16 = reshape(shape = var_914, x = var_913_cast_fp16)[name = string("q_19_cast_fp16")];
+            tensor<int32, [4]> var_916_begin_0 = const()[name = string("op_916_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_916_end_0 = const()[name = string("op_916_end_0"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<bool, [4]> var_916_end_mask_0 = const()[name = string("op_916_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_916_squeeze_mask_0 = const()[name = string("op_916_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [1024, 32, 128]> var_916_cast_fp16 = slice_by_index(begin = var_916_begin_0, end = var_916_end_0, end_mask = var_916_end_mask_0, squeeze_mask = var_916_squeeze_mask_0, x = k_13_cast_fp16)[name = string("op_916_cast_fp16")];
+            tensor<int32, [3]> var_917 = const()[name = string("op_917"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_919 = const()[name = string("op_919"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<fp16, [32, 1024, 128]> var_918_cast_fp16 = transpose(perm = var_917, x = var_916_cast_fp16)[name = string("transpose_44")];
+            tensor<fp16, [32, 16, 64, 128]> k_15_cast_fp16 = reshape(shape = var_919, x = var_918_cast_fp16)[name = string("k_15_cast_fp16")];
+            tensor<int32, [4]> var_921_begin_0 = const()[name = string("op_921_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_921_end_0 = const()[name = string("op_921_end_0"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<bool, [4]> var_921_end_mask_0 = const()[name = string("op_921_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_921_squeeze_mask_0 = const()[name = string("op_921_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [1024, 32, 128]> var_921_cast_fp16 = slice_by_index(begin = var_921_begin_0, end = var_921_end_0, end_mask = var_921_end_mask_0, squeeze_mask = var_921_squeeze_mask_0, x = v_5_cast_fp16)[name = string("op_921_cast_fp16")];
+            tensor<int32, [3]> var_922 = const()[name = string("op_922"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_924 = const()[name = string("op_924"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<fp16, [32, 1024, 128]> var_923_cast_fp16 = transpose(perm = var_922, x = var_921_cast_fp16)[name = string("transpose_43")];
+            tensor<fp16, [32, 16, 64, 128]> v_7_cast_fp16 = reshape(shape = var_924, x = var_923_cast_fp16)[name = string("v_7_cast_fp16")];
+            tensor<int32, [3]> var_926_begin_0 = const()[name = string("op_926_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_926_end_0 = const()[name = string("op_926_end_0"), val = tensor<int32, [3]>([1, 1024, 32])];
+            tensor<bool, [3]> var_926_end_mask_0 = const()[name = string("op_926_end_mask_0"), val = tensor<bool, [3]>([false, true, true])];
+            tensor<bool, [3]> var_926_squeeze_mask_0 = const()[name = string("op_926_squeeze_mask_0"), val = tensor<bool, [3]>([true, false, false])];
+            tensor<fp16, [1024, 32]> var_926_cast_fp16 = slice_by_index(begin = var_926_begin_0, end = var_926_end_0, end_mask = var_926_end_mask_0, squeeze_mask = var_926_squeeze_mask_0, x = beta_5_cast_fp16)[name = string("op_926_cast_fp16")];
+            tensor<int32, [2]> var_927_perm_0 = const()[name = string("op_927_perm_0"), val = tensor<int32, [2]>([1, 0])];
+            tensor<int32, [4]> var_928 = const()[name = string("op_928"), val = tensor<int32, [4]>([32, 16, 64, 1])];
+            tensor<fp16, [32, 1024]> var_927_cast_fp16 = transpose(perm = var_927_perm_0, x = var_926_cast_fp16)[name = string("transpose_42")];
+            tensor<fp16, [32, 16, 64, 1]> beta_7_cast_fp16 = reshape(shape = var_928, x = var_927_cast_fp16)[name = string("beta_7_cast_fp16")];
+            tensor<int32, [3]> var_930_begin_0 = const()[name = string("op_930_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_930_end_0 = const()[name = string("op_930_end_0"), val = tensor<int32, [3]>([1, 1024, 32])];
+            tensor<bool, [3]> var_930_end_mask_0 = const()[name = string("op_930_end_mask_0"), val = tensor<bool, [3]>([false, true, true])];
+            tensor<bool, [3]> var_930_squeeze_mask_0 = const()[name = string("op_930_squeeze_mask_0"), val = tensor<bool, [3]>([true, false, false])];
+            tensor<fp16, [1024, 32]> var_930_cast_fp16 = slice_by_index(begin = var_930_begin_0, end = var_930_end_0, end_mask = var_930_end_mask_0, squeeze_mask = var_930_squeeze_mask_0, x = g_5_cast_fp16)[name = string("op_930_cast_fp16")];
+            tensor<int32, [2]> var_931_perm_0 = const()[name = string("op_931_perm_0"), val = tensor<int32, [2]>([1, 0])];
+            tensor<int32, [3]> var_932 = const()[name = string("op_932"), val = tensor<int32, [3]>([32, 16, 64])];
+            tensor<fp16, [32, 1024]> var_931_cast_fp16 = transpose(perm = var_931_perm_0, x = var_930_cast_fp16)[name = string("transpose_41")];
+            tensor<fp16, [32, 16, 64]> g_7_cast_fp16 = reshape(shape = var_932, x = var_931_cast_fp16)[name = string("g_7_cast_fp16")];
+            tensor<int32, [1]> g_col_3_axes_0 = const()[name = string("g_col_3_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 16, 64, 1]> g_col_3_cast_fp16 = expand_dims(axes = g_col_3_axes_0, x = g_7_cast_fp16)[name = string("g_col_3_cast_fp16")];
+            bool var_935_transpose_x_0 = const()[name = string("op_935_transpose_x_0"), val = bool(false)];
+            bool var_935_transpose_y_0 = const()[name = string("op_935_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 16, 64, 1]> var_935_cast_fp16 = matmul(transpose_x = var_935_transpose_x_0, transpose_y = var_935_transpose_y_0, x = layers_0_linear_attn_lower_incl_to_fp16, y = g_col_3_cast_fp16)[name = string("op_935_cast_fp16")];
+            tensor<int32, [1]> cum_3_axes_0 = const()[name = string("cum_3_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 16, 64]> cum_3_cast_fp16 = squeeze(axes = cum_3_axes_0, x = var_935_cast_fp16)[name = string("cum_3_cast_fp16")];
+            bool var_937_transpose_x_0 = const()[name = string("op_937_transpose_x_0"), val = bool(false)];
+            bool var_937_transpose_y_0 = const()[name = string("op_937_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 16, 64, 1]> var_937_cast_fp16 = matmul(transpose_x = var_937_transpose_x_0, transpose_y = var_937_transpose_y_0, x = layers_0_linear_attn_suffix_sum_t_to_fp16, y = g_col_3_cast_fp16)[name = string("op_937_cast_fp16")];
+            tensor<int32, [1]> to_end_3_axes_0 = const()[name = string("to_end_3_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 16, 64]> to_end_3_cast_fp16 = squeeze(axes = to_end_3_axes_0, x = var_937_cast_fp16)[name = string("to_end_3_cast_fp16")];
+            bool var_939_transpose_x_0 = const()[name = string("op_939_transpose_x_0"), val = bool(false)];
+            bool var_939_transpose_y_0 = const()[name = string("op_939_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 16, 4096, 1]> var_939_cast_fp16 = matmul(transpose_x = var_939_transpose_x_0, transpose_y = var_939_transpose_y_0, x = layers_0_linear_attn_pair_sum_t_to_fp16, y = g_col_3_cast_fp16)[name = string("op_939_cast_fp16")];
+            tensor<int32, [4]> var_940 = const()[name = string("op_940"), val = tensor<int32, [4]>([32, 16, 64, 64])];
+            tensor<fp16, [32, 16, 64, 64]> pair_15_cast_fp16 = reshape(shape = var_940, x = var_939_cast_fp16)[name = string("pair_15_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 64]> var_942_cast_fp16 = exp(x = pair_15_cast_fp16)[name = string("op_942_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 64]> pair_decay_3_cast_fp16 = mul(x = var_942_cast_fp16, y = layers_0_linear_attn_lower_incl_to_fp16)[name = string("pair_decay_3_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 128]> k_beta_3_cast_fp16 = mul(x = k_15_cast_fp16, y = beta_7_cast_fp16)[name = string("k_beta_3_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 128]> v_beta_3_cast_fp16 = mul(x = v_7_cast_fp16, y = beta_7_cast_fp16)[name = string("v_beta_3_cast_fp16")];
+            bool var_947_transpose_x_1 = const()[name = string("op_947_transpose_x_1"), val = bool(false)];
+            bool var_947_transpose_y_1 = const()[name = string("op_947_transpose_y_1"), val = bool(true)];
+            tensor<fp16, [32, 16, 64, 64]> var_947_cast_fp16 = matmul(transpose_x = var_947_transpose_x_1, transpose_y = var_947_transpose_y_1, x = k_beta_3_cast_fp16, y = k_15_cast_fp16)[name = string("op_947_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 64]> kkt_3_cast_fp16 = mul(x = var_947_cast_fp16, y = pair_decay_3_cast_fp16)[name = string("kkt_3_cast_fp16")];
+            bool var_950_transpose_x_1 = const()[name = string("op_950_transpose_x_1"), val = bool(false)];
+            bool var_950_transpose_y_1 = const()[name = string("op_950_transpose_y_1"), val = bool(true)];
+            tensor<fp16, [32, 16, 64, 64]> var_950_cast_fp16 = matmul(transpose_x = var_950_transpose_x_1, transpose_y = var_950_transpose_y_1, x = q_19_cast_fp16, y = k_15_cast_fp16)[name = string("op_950_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 64]> attn_intra_3_cast_fp16 = mul(x = var_950_cast_fp16, y = pair_decay_3_cast_fp16)[name = string("attn_intra_3_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 64]> var_952_cast_fp16 = mul(x = kkt_3_cast_fp16, y = layers_0_linear_attn_strict_lower_to_fp16)[name = string("op_952_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 64]> t_5_cast_fp16 = add(x = layers_0_linear_attn_eye_to_fp16, y = var_952_cast_fp16)[name = string("t_5_cast_fp16")];
+            tensor<int32, [3]> var_956 = const()[name = string("op_956"), val = tensor<int32, [3]>([-1, 64, 64])];
+            tensor<fp16, [512, 64, 64]> t_7_cast_fp16 = reshape(shape = var_956, x = t_5_cast_fp16)[name = string("t_7_cast_fp16")];
+            tensor<int32, [5]> var_961 = const()[name = string("op_961"), val = tensor<int32, [5]>([512, 32, 2, 32, 2])];
+            tensor<fp16, [512, 32, 2, 32, 2]> var_962_cast_fp16 = reshape(shape = var_961, x = t_7_cast_fp16)[name = string("op_962_cast_fp16")];
+            tensor<fp16, [512, 32, 2, 32, 2]> var_967_cast_fp16 = mul(x = var_962_cast_fp16, y = var_213_to_fp16)[name = string("op_967_cast_fp16")];
+            tensor<int32, [1]> blocks_13_axes_0 = const()[name = string("blocks_13_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_13_keep_dims_0 = const()[name = string("blocks_13_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 32, 2, 2]> blocks_13_cast_fp16 = reduce_sum(axes = blocks_13_axes_0, keep_dims = blocks_13_keep_dims_0, x = var_967_cast_fp16)[name = string("blocks_13_cast_fp16")];
+            tensor<int32, [4]> var_970_begin_0 = const()[name = string("op_970_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
+            tensor<int32, [4]> var_970_end_0 = const()[name = string("op_970_end_0"), val = tensor<int32, [4]>([512, 32, 2, 2])];
+            tensor<bool, [4]> var_970_end_mask_0 = const()[name = string("op_970_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 32, 1, 2]> var_970_cast_fp16 = slice_by_index(begin = var_970_begin_0, end = var_970_end_0, end_mask = var_970_end_mask_0, x = blocks_13_cast_fp16)[name = string("op_970_cast_fp16")];
+            tensor<int32, [4]> x_31_begin_0 = const()[name = string("x_31_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_31_end_0 = const()[name = string("x_31_end_0"), val = tensor<int32, [4]>([512, 32, 1, 1])];
+            tensor<bool, [4]> x_31_end_mask_0 = const()[name = string("x_31_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 32, 1, 1]> x_31_cast_fp16 = slice_by_index(begin = x_31_begin_0, end = x_31_end_0, end_mask = x_31_end_mask_0, x = var_970_cast_fp16)[name = string("x_31_cast_fp16")];
+            bool var_980_transpose_x_0 = const()[name = string("op_980_transpose_x_0"), val = bool(false)];
+            bool var_980_transpose_y_0 = const()[name = string("op_980_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 32, 1, 1]> var_980_cast_fp16 = matmul(transpose_x = var_980_transpose_x_0, transpose_y = var_980_transpose_y_0, x = x_31_cast_fp16, y = var_221_to_fp16)[name = string("op_980_cast_fp16")];
+            bool var_981_transpose_x_0 = const()[name = string("op_981_transpose_x_0"), val = bool(false)];
+            bool var_981_transpose_y_0 = const()[name = string("op_981_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 32, 1, 1]> var_981_cast_fp16 = matmul(transpose_x = var_981_transpose_x_0, transpose_y = var_981_transpose_y_0, x = var_221_to_fp16, y = var_980_cast_fp16)[name = string("op_981_cast_fp16")];
+            fp16 const_18_promoted_to_fp16 = const()[name = string("const_18_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 32, 1, 1]> lower_13_cast_fp16 = mul(x = var_981_cast_fp16, y = const_18_promoted_to_fp16)[name = string("lower_13_cast_fp16")];
+            bool bottom_13_interleave_0 = const()[name = string("bottom_13_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 32, 1, 2]> bottom_13_cast_fp16 = concat(axis = var_802, interleave = bottom_13_interleave_0, values = (lower_13_cast_fp16, var_221_to_fp16))[name = string("bottom_13_cast_fp16")];
+            bool inv_19_interleave_0 = const()[name = string("inv_19_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 32, 2, 2]> inv_19_cast_fp16 = concat(axis = var_787, interleave = inv_19_interleave_0, values = (top_1_to_fp16, bottom_13_cast_fp16))[name = string("inv_19_cast_fp16")];
+            tensor<int32, [5]> var_990 = const()[name = string("op_990"), val = tensor<int32, [5]>([512, 16, 4, 16, 4])];
+            tensor<fp16, [512, 16, 4, 16, 4]> var_991_cast_fp16 = reshape(shape = var_990, x = t_7_cast_fp16)[name = string("op_991_cast_fp16")];
+            tensor<fp16, [512, 16, 4, 16, 4]> var_996_cast_fp16 = mul(x = var_991_cast_fp16, y = var_242_to_fp16)[name = string("op_996_cast_fp16")];
+            tensor<int32, [1]> blocks_15_axes_0 = const()[name = string("blocks_15_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_15_keep_dims_0 = const()[name = string("blocks_15_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 4, 4]> blocks_15_cast_fp16 = reduce_sum(axes = blocks_15_axes_0, keep_dims = blocks_15_keep_dims_0, x = var_996_cast_fp16)[name = string("blocks_15_cast_fp16")];
+            tensor<int32, [4]> var_999_begin_0 = const()[name = string("op_999_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
+            tensor<int32, [4]> var_999_end_0 = const()[name = string("op_999_end_0"), val = tensor<int32, [4]>([512, 16, 4, 4])];
+            tensor<bool, [4]> var_999_end_mask_0 = const()[name = string("op_999_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 16, 2, 4]> var_999_cast_fp16 = slice_by_index(begin = var_999_begin_0, end = var_999_end_0, end_mask = var_999_end_mask_0, x = blocks_15_cast_fp16)[name = string("op_999_cast_fp16")];
+            tensor<int32, [4]> x_33_begin_0 = const()[name = string("x_33_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_33_end_0 = const()[name = string("x_33_end_0"), val = tensor<int32, [4]>([512, 16, 2, 2])];
+            tensor<bool, [4]> x_33_end_mask_0 = const()[name = string("x_33_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 16, 2, 2]> x_33_cast_fp16 = slice_by_index(begin = x_33_begin_0, end = x_33_end_0, end_mask = x_33_end_mask_0, x = var_999_cast_fp16)[name = string("x_33_cast_fp16")];
+            tensor<int32, [5]> var_1001 = const()[name = string("op_1001"), val = tensor<int32, [5]>([512, 16, 2, 2, 2])];
+            tensor<fp16, [512, 16, 2, 2, 2]> pair_19_cast_fp16 = reshape(shape = var_1001, x = inv_19_cast_fp16)[name = string("pair_19_cast_fp16")];
+            tensor<int32, [5]> var_1003_begin_0 = const()[name = string("op_1003_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_1003_end_0 = const()[name = string("op_1003_end_0"), val = tensor<int32, [5]>([512, 16, 1, 2, 2])];
+            tensor<bool, [5]> var_1003_end_mask_0 = const()[name = string("op_1003_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_1003_squeeze_mask_0 = const()[name = string("op_1003_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 16, 2, 2]> var_1003_cast_fp16 = slice_by_index(begin = var_1003_begin_0, end = var_1003_end_0, end_mask = var_1003_end_mask_0, squeeze_mask = var_1003_squeeze_mask_0, x = pair_19_cast_fp16)[name = string("op_1003_cast_fp16")];
+            tensor<int32, [5]> var_1006_begin_0 = const()[name = string("op_1006_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_1006_end_0 = const()[name = string("op_1006_end_0"), val = tensor<int32, [5]>([512, 16, 2, 2, 2])];
+            tensor<bool, [5]> var_1006_end_mask_0 = const()[name = string("op_1006_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_1006_squeeze_mask_0 = const()[name = string("op_1006_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 16, 2, 2]> var_1006_cast_fp16 = slice_by_index(begin = var_1006_begin_0, end = var_1006_end_0, end_mask = var_1006_end_mask_0, squeeze_mask = var_1006_squeeze_mask_0, x = pair_19_cast_fp16)[name = string("op_1006_cast_fp16")];
+            bool var_1009_transpose_x_0 = const()[name = string("op_1009_transpose_x_0"), val = bool(false)];
+            bool var_1009_transpose_y_0 = const()[name = string("op_1009_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 2, 2]> var_1009_cast_fp16 = matmul(transpose_x = var_1009_transpose_x_0, transpose_y = var_1009_transpose_y_0, x = x_33_cast_fp16, y = var_1003_cast_fp16)[name = string("op_1009_cast_fp16")];
+            bool var_1010_transpose_x_0 = const()[name = string("op_1010_transpose_x_0"), val = bool(false)];
+            bool var_1010_transpose_y_0 = const()[name = string("op_1010_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 2, 2]> var_1010_cast_fp16 = matmul(transpose_x = var_1010_transpose_x_0, transpose_y = var_1010_transpose_y_0, x = var_1006_cast_fp16, y = var_1009_cast_fp16)[name = string("op_1010_cast_fp16")];
+            fp16 const_19_promoted_to_fp16 = const()[name = string("const_19_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 16, 2, 2]> lower_15_cast_fp16 = mul(x = var_1010_cast_fp16, y = const_19_promoted_to_fp16)[name = string("lower_15_cast_fp16")];
+            bool top_15_interleave_0 = const()[name = string("top_15_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 2, 4]> top_15_cast_fp16 = concat(axis = var_802, interleave = top_15_interleave_0, values = (var_1003_cast_fp16, var_259_to_fp16))[name = string("top_15_cast_fp16")];
+            bool bottom_15_interleave_0 = const()[name = string("bottom_15_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 2, 4]> bottom_15_cast_fp16 = concat(axis = var_802, interleave = bottom_15_interleave_0, values = (lower_15_cast_fp16, var_1006_cast_fp16))[name = string("bottom_15_cast_fp16")];
+            bool inv_21_interleave_0 = const()[name = string("inv_21_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 4, 4]> inv_21_cast_fp16 = concat(axis = var_787, interleave = inv_21_interleave_0, values = (top_15_cast_fp16, bottom_15_cast_fp16))[name = string("inv_21_cast_fp16")];
+            tensor<int32, [5]> var_1019 = const()[name = string("op_1019"), val = tensor<int32, [5]>([512, 8, 8, 8, 8])];
+            tensor<fp16, [512, 8, 8, 8, 8]> var_1020_cast_fp16 = reshape(shape = var_1019, x = t_7_cast_fp16)[name = string("op_1020_cast_fp16")];
+            tensor<fp16, [8, 1, 8, 1]> var_1024_to_fp16 = const()[name = string("op_1024_to_fp16"), val = tensor<fp16, [8, 1, 8, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(685602368)))];
+            tensor<fp16, [512, 8, 8, 8, 8]> var_1025_cast_fp16 = mul(x = var_1020_cast_fp16, y = var_1024_to_fp16)[name = string("op_1025_cast_fp16")];
+            tensor<int32, [1]> blocks_17_axes_0 = const()[name = string("blocks_17_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_17_keep_dims_0 = const()[name = string("blocks_17_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 8, 8]> blocks_17_cast_fp16 = reduce_sum(axes = blocks_17_axes_0, keep_dims = blocks_17_keep_dims_0, x = var_1025_cast_fp16)[name = string("blocks_17_cast_fp16")];
+            tensor<int32, [4]> var_1028_begin_0 = const()[name = string("op_1028_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
+            tensor<int32, [4]> var_1028_end_0 = const()[name = string("op_1028_end_0"), val = tensor<int32, [4]>([512, 8, 8, 8])];
+            tensor<bool, [4]> var_1028_end_mask_0 = const()[name = string("op_1028_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 8, 4, 8]> var_1028_cast_fp16 = slice_by_index(begin = var_1028_begin_0, end = var_1028_end_0, end_mask = var_1028_end_mask_0, x = blocks_17_cast_fp16)[name = string("op_1028_cast_fp16")];
+            tensor<int32, [4]> x_35_begin_0 = const()[name = string("x_35_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_35_end_0 = const()[name = string("x_35_end_0"), val = tensor<int32, [4]>([512, 8, 4, 4])];
+            tensor<bool, [4]> x_35_end_mask_0 = const()[name = string("x_35_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 8, 4, 4]> x_35_cast_fp16 = slice_by_index(begin = x_35_begin_0, end = x_35_end_0, end_mask = x_35_end_mask_0, x = var_1028_cast_fp16)[name = string("x_35_cast_fp16")];
+            tensor<int32, [5]> var_1030 = const()[name = string("op_1030"), val = tensor<int32, [5]>([512, 8, 2, 4, 4])];
+            tensor<fp16, [512, 8, 2, 4, 4]> pair_21_cast_fp16 = reshape(shape = var_1030, x = inv_21_cast_fp16)[name = string("pair_21_cast_fp16")];
+            tensor<int32, [5]> var_1032_begin_0 = const()[name = string("op_1032_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_1032_end_0 = const()[name = string("op_1032_end_0"), val = tensor<int32, [5]>([512, 8, 1, 4, 4])];
+            tensor<bool, [5]> var_1032_end_mask_0 = const()[name = string("op_1032_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_1032_squeeze_mask_0 = const()[name = string("op_1032_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 8, 4, 4]> var_1032_cast_fp16 = slice_by_index(begin = var_1032_begin_0, end = var_1032_end_0, end_mask = var_1032_end_mask_0, squeeze_mask = var_1032_squeeze_mask_0, x = pair_21_cast_fp16)[name = string("op_1032_cast_fp16")];
+            tensor<int32, [5]> var_1035_begin_0 = const()[name = string("op_1035_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_1035_end_0 = const()[name = string("op_1035_end_0"), val = tensor<int32, [5]>([512, 8, 2, 4, 4])];
+            tensor<bool, [5]> var_1035_end_mask_0 = const()[name = string("op_1035_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_1035_squeeze_mask_0 = const()[name = string("op_1035_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 8, 4, 4]> var_1035_cast_fp16 = slice_by_index(begin = var_1035_begin_0, end = var_1035_end_0, end_mask = var_1035_end_mask_0, squeeze_mask = var_1035_squeeze_mask_0, x = pair_21_cast_fp16)[name = string("op_1035_cast_fp16")];
+            bool var_1038_transpose_x_0 = const()[name = string("op_1038_transpose_x_0"), val = bool(false)];
+            bool var_1038_transpose_y_0 = const()[name = string("op_1038_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 4, 4]> var_1038_cast_fp16 = matmul(transpose_x = var_1038_transpose_x_0, transpose_y = var_1038_transpose_y_0, x = x_35_cast_fp16, y = var_1032_cast_fp16)[name = string("op_1038_cast_fp16")];
+            bool var_1039_transpose_x_0 = const()[name = string("op_1039_transpose_x_0"), val = bool(false)];
+            bool var_1039_transpose_y_0 = const()[name = string("op_1039_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 4, 4]> var_1039_cast_fp16 = matmul(transpose_x = var_1039_transpose_x_0, transpose_y = var_1039_transpose_y_0, x = var_1035_cast_fp16, y = var_1038_cast_fp16)[name = string("op_1039_cast_fp16")];
+            fp16 const_20_promoted_to_fp16 = const()[name = string("const_20_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 8, 4, 4]> lower_17_cast_fp16 = mul(x = var_1039_cast_fp16, y = const_20_promoted_to_fp16)[name = string("lower_17_cast_fp16")];
+            bool top_17_interleave_0 = const()[name = string("top_17_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 4, 8]> top_17_cast_fp16 = concat(axis = var_802, interleave = top_17_interleave_0, values = (var_1032_cast_fp16, var_288_to_fp16))[name = string("top_17_cast_fp16")];
+            bool bottom_17_interleave_0 = const()[name = string("bottom_17_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 4, 8]> bottom_17_cast_fp16 = concat(axis = var_802, interleave = bottom_17_interleave_0, values = (lower_17_cast_fp16, var_1035_cast_fp16))[name = string("bottom_17_cast_fp16")];
+            bool inv_23_interleave_0 = const()[name = string("inv_23_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 8, 8]> inv_23_cast_fp16 = concat(axis = var_787, interleave = inv_23_interleave_0, values = (top_17_cast_fp16, bottom_17_cast_fp16))[name = string("inv_23_cast_fp16")];
+            tensor<int32, [5]> var_1048 = const()[name = string("op_1048"), val = tensor<int32, [5]>([512, 4, 16, 4, 16])];
+            tensor<fp16, [512, 4, 16, 4, 16]> var_1049_cast_fp16 = reshape(shape = var_1048, x = t_7_cast_fp16)[name = string("op_1049_cast_fp16")];
+            tensor<fp16, [4, 1, 4, 1]> var_1053_to_fp16 = const()[name = string("op_1053_to_fp16"), val = tensor<fp16, [4, 1, 4, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(685602560)))];
+            tensor<fp16, [512, 4, 16, 4, 16]> var_1054_cast_fp16 = mul(x = var_1049_cast_fp16, y = var_1053_to_fp16)[name = string("op_1054_cast_fp16")];
+            tensor<int32, [1]> blocks_19_axes_0 = const()[name = string("blocks_19_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_19_keep_dims_0 = const()[name = string("blocks_19_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 16, 16]> blocks_19_cast_fp16 = reduce_sum(axes = blocks_19_axes_0, keep_dims = blocks_19_keep_dims_0, x = var_1054_cast_fp16)[name = string("blocks_19_cast_fp16")];
+            tensor<int32, [4]> var_1057_begin_0 = const()[name = string("op_1057_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
+            tensor<int32, [4]> var_1057_end_0 = const()[name = string("op_1057_end_0"), val = tensor<int32, [4]>([512, 4, 16, 16])];
+            tensor<bool, [4]> var_1057_end_mask_0 = const()[name = string("op_1057_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 4, 8, 16]> var_1057_cast_fp16 = slice_by_index(begin = var_1057_begin_0, end = var_1057_end_0, end_mask = var_1057_end_mask_0, x = blocks_19_cast_fp16)[name = string("op_1057_cast_fp16")];
+            tensor<int32, [4]> x_37_begin_0 = const()[name = string("x_37_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_37_end_0 = const()[name = string("x_37_end_0"), val = tensor<int32, [4]>([512, 4, 8, 8])];
+            tensor<bool, [4]> x_37_end_mask_0 = const()[name = string("x_37_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 4, 8, 8]> x_37_cast_fp16 = slice_by_index(begin = x_37_begin_0, end = x_37_end_0, end_mask = x_37_end_mask_0, x = var_1057_cast_fp16)[name = string("x_37_cast_fp16")];
+            tensor<int32, [5]> var_1059 = const()[name = string("op_1059"), val = tensor<int32, [5]>([512, 4, 2, 8, 8])];
+            tensor<fp16, [512, 4, 2, 8, 8]> pair_23_cast_fp16 = reshape(shape = var_1059, x = inv_23_cast_fp16)[name = string("pair_23_cast_fp16")];
+            tensor<int32, [5]> var_1061_begin_0 = const()[name = string("op_1061_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_1061_end_0 = const()[name = string("op_1061_end_0"), val = tensor<int32, [5]>([512, 4, 1, 8, 8])];
+            tensor<bool, [5]> var_1061_end_mask_0 = const()[name = string("op_1061_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_1061_squeeze_mask_0 = const()[name = string("op_1061_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 4, 8, 8]> var_1061_cast_fp16 = slice_by_index(begin = var_1061_begin_0, end = var_1061_end_0, end_mask = var_1061_end_mask_0, squeeze_mask = var_1061_squeeze_mask_0, x = pair_23_cast_fp16)[name = string("op_1061_cast_fp16")];
+            tensor<int32, [5]> var_1064_begin_0 = const()[name = string("op_1064_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_1064_end_0 = const()[name = string("op_1064_end_0"), val = tensor<int32, [5]>([512, 4, 2, 8, 8])];
+            tensor<bool, [5]> var_1064_end_mask_0 = const()[name = string("op_1064_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_1064_squeeze_mask_0 = const()[name = string("op_1064_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 4, 8, 8]> var_1064_cast_fp16 = slice_by_index(begin = var_1064_begin_0, end = var_1064_end_0, end_mask = var_1064_end_mask_0, squeeze_mask = var_1064_squeeze_mask_0, x = pair_23_cast_fp16)[name = string("op_1064_cast_fp16")];
+            bool var_1067_transpose_x_0 = const()[name = string("op_1067_transpose_x_0"), val = bool(false)];
+            bool var_1067_transpose_y_0 = const()[name = string("op_1067_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 8, 8]> var_1067_cast_fp16 = matmul(transpose_x = var_1067_transpose_x_0, transpose_y = var_1067_transpose_y_0, x = x_37_cast_fp16, y = var_1061_cast_fp16)[name = string("op_1067_cast_fp16")];
+            bool var_1068_transpose_x_0 = const()[name = string("op_1068_transpose_x_0"), val = bool(false)];
+            bool var_1068_transpose_y_0 = const()[name = string("op_1068_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 8, 8]> var_1068_cast_fp16 = matmul(transpose_x = var_1068_transpose_x_0, transpose_y = var_1068_transpose_y_0, x = var_1064_cast_fp16, y = var_1067_cast_fp16)[name = string("op_1068_cast_fp16")];
+            fp16 const_21_promoted_to_fp16 = const()[name = string("const_21_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 4, 8, 8]> lower_19_cast_fp16 = mul(x = var_1068_cast_fp16, y = const_21_promoted_to_fp16)[name = string("lower_19_cast_fp16")];
+            bool top_19_interleave_0 = const()[name = string("top_19_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 8, 16]> top_19_cast_fp16 = concat(axis = var_802, interleave = top_19_interleave_0, values = (var_1061_cast_fp16, var_317_to_fp16))[name = string("top_19_cast_fp16")];
+            bool bottom_19_interleave_0 = const()[name = string("bottom_19_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 8, 16]> bottom_19_cast_fp16 = concat(axis = var_802, interleave = bottom_19_interleave_0, values = (lower_19_cast_fp16, var_1064_cast_fp16))[name = string("bottom_19_cast_fp16")];
+            bool inv_25_interleave_0 = const()[name = string("inv_25_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 16, 16]> inv_25_cast_fp16 = concat(axis = var_787, interleave = inv_25_interleave_0, values = (top_19_cast_fp16, bottom_19_cast_fp16))[name = string("inv_25_cast_fp16")];
+            tensor<int32, [5]> var_1077 = const()[name = string("op_1077"), val = tensor<int32, [5]>([512, 2, 32, 2, 32])];
+            tensor<fp16, [512, 2, 32, 2, 32]> var_1078_cast_fp16 = reshape(shape = var_1077, x = t_7_cast_fp16)[name = string("op_1078_cast_fp16")];
+            tensor<fp16, [2, 1, 2, 1]> var_1082_to_fp16 = const()[name = string("op_1082_to_fp16"), val = tensor<fp16, [2, 1, 2, 1]>([[[[0x1p+0], [0x0p+0]]], [[[0x0p+0], [0x1p+0]]]])];
+            tensor<fp16, [512, 2, 32, 2, 32]> var_1083_cast_fp16 = mul(x = var_1078_cast_fp16, y = var_1082_to_fp16)[name = string("op_1083_cast_fp16")];
+            tensor<int32, [1]> blocks_21_axes_0 = const()[name = string("blocks_21_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_21_keep_dims_0 = const()[name = string("blocks_21_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 32, 32]> blocks_21_cast_fp16 = reduce_sum(axes = blocks_21_axes_0, keep_dims = blocks_21_keep_dims_0, x = var_1083_cast_fp16)[name = string("blocks_21_cast_fp16")];
+            tensor<int32, [4]> var_1086_begin_0 = const()[name = string("op_1086_begin_0"), val = tensor<int32, [4]>([0, 0, 16, 0])];
+            tensor<int32, [4]> var_1086_end_0 = const()[name = string("op_1086_end_0"), val = tensor<int32, [4]>([512, 2, 32, 32])];
+            tensor<bool, [4]> var_1086_end_mask_0 = const()[name = string("op_1086_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 2, 16, 32]> var_1086_cast_fp16 = slice_by_index(begin = var_1086_begin_0, end = var_1086_end_0, end_mask = var_1086_end_mask_0, x = blocks_21_cast_fp16)[name = string("op_1086_cast_fp16")];
+            tensor<int32, [4]> x_39_begin_0 = const()[name = string("x_39_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_39_end_0 = const()[name = string("x_39_end_0"), val = tensor<int32, [4]>([512, 2, 16, 16])];
+            tensor<bool, [4]> x_39_end_mask_0 = const()[name = string("x_39_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 2, 16, 16]> x_39_cast_fp16 = slice_by_index(begin = x_39_begin_0, end = x_39_end_0, end_mask = x_39_end_mask_0, x = var_1086_cast_fp16)[name = string("x_39_cast_fp16")];
+            tensor<int32, [5]> var_1088 = const()[name = string("op_1088"), val = tensor<int32, [5]>([512, 2, 2, 16, 16])];
+            tensor<fp16, [512, 2, 2, 16, 16]> pair_25_cast_fp16 = reshape(shape = var_1088, x = inv_25_cast_fp16)[name = string("pair_25_cast_fp16")];
+            tensor<int32, [5]> var_1090_begin_0 = const()[name = string("op_1090_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_1090_end_0 = const()[name = string("op_1090_end_0"), val = tensor<int32, [5]>([512, 2, 1, 16, 16])];
+            tensor<bool, [5]> var_1090_end_mask_0 = const()[name = string("op_1090_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_1090_squeeze_mask_0 = const()[name = string("op_1090_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 2, 16, 16]> var_1090_cast_fp16 = slice_by_index(begin = var_1090_begin_0, end = var_1090_end_0, end_mask = var_1090_end_mask_0, squeeze_mask = var_1090_squeeze_mask_0, x = pair_25_cast_fp16)[name = string("op_1090_cast_fp16")];
+            tensor<int32, [5]> var_1093_begin_0 = const()[name = string("op_1093_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_1093_end_0 = const()[name = string("op_1093_end_0"), val = tensor<int32, [5]>([512, 2, 2, 16, 16])];
+            tensor<bool, [5]> var_1093_end_mask_0 = const()[name = string("op_1093_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_1093_squeeze_mask_0 = const()[name = string("op_1093_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 2, 16, 16]> var_1093_cast_fp16 = slice_by_index(begin = var_1093_begin_0, end = var_1093_end_0, end_mask = var_1093_end_mask_0, squeeze_mask = var_1093_squeeze_mask_0, x = pair_25_cast_fp16)[name = string("op_1093_cast_fp16")];
+            bool var_1096_transpose_x_0 = const()[name = string("op_1096_transpose_x_0"), val = bool(false)];
+            bool var_1096_transpose_y_0 = const()[name = string("op_1096_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 16, 16]> var_1096_cast_fp16 = matmul(transpose_x = var_1096_transpose_x_0, transpose_y = var_1096_transpose_y_0, x = x_39_cast_fp16, y = var_1090_cast_fp16)[name = string("op_1096_cast_fp16")];
+            bool var_1097_transpose_x_0 = const()[name = string("op_1097_transpose_x_0"), val = bool(false)];
+            bool var_1097_transpose_y_0 = const()[name = string("op_1097_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 16, 16]> var_1097_cast_fp16 = matmul(transpose_x = var_1097_transpose_x_0, transpose_y = var_1097_transpose_y_0, x = var_1093_cast_fp16, y = var_1096_cast_fp16)[name = string("op_1097_cast_fp16")];
+            fp16 const_22_promoted_to_fp16 = const()[name = string("const_22_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 2, 16, 16]> lower_21_cast_fp16 = mul(x = var_1097_cast_fp16, y = const_22_promoted_to_fp16)[name = string("lower_21_cast_fp16")];
+            bool top_21_interleave_0 = const()[name = string("top_21_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 16, 32]> top_21_cast_fp16 = concat(axis = var_802, interleave = top_21_interleave_0, values = (var_1090_cast_fp16, var_346_to_fp16))[name = string("top_21_cast_fp16")];
+            bool bottom_21_interleave_0 = const()[name = string("bottom_21_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 16, 32]> bottom_21_cast_fp16 = concat(axis = var_802, interleave = bottom_21_interleave_0, values = (lower_21_cast_fp16, var_1093_cast_fp16))[name = string("bottom_21_cast_fp16")];
+            bool inv_27_interleave_0 = const()[name = string("inv_27_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 32, 32]> inv_27_cast_fp16 = concat(axis = var_787, interleave = inv_27_interleave_0, values = (top_21_cast_fp16, bottom_21_cast_fp16))[name = string("inv_27_cast_fp16")];
+            tensor<int32, [5]> var_1106 = const()[name = string("op_1106"), val = tensor<int32, [5]>([512, 1, 64, 1, 64])];
+            tensor<fp16, [512, 1, 64, 1, 64]> var_1107_cast_fp16 = reshape(shape = var_1106, x = t_7_cast_fp16)[name = string("op_1107_cast_fp16")];
+            tensor<int32, [1]> blocks_23_axes_0 = const()[name = string("blocks_23_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_23_keep_dims_0 = const()[name = string("blocks_23_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 64, 64]> blocks_23_cast_fp16 = reduce_sum(axes = blocks_23_axes_0, keep_dims = blocks_23_keep_dims_0, x = var_1107_cast_fp16)[name = string("blocks_23_cast_fp16")];
+            tensor<int32, [4]> var_1115_begin_0 = const()[name = string("op_1115_begin_0"), val = tensor<int32, [4]>([0, 0, 32, 0])];
+            tensor<int32, [4]> var_1115_end_0 = const()[name = string("op_1115_end_0"), val = tensor<int32, [4]>([512, 1, 64, 64])];
+            tensor<bool, [4]> var_1115_end_mask_0 = const()[name = string("op_1115_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 1, 32, 64]> var_1115_cast_fp16 = slice_by_index(begin = var_1115_begin_0, end = var_1115_end_0, end_mask = var_1115_end_mask_0, x = blocks_23_cast_fp16)[name = string("op_1115_cast_fp16")];
+            tensor<int32, [4]> x_41_begin_0 = const()[name = string("x_41_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_41_end_0 = const()[name = string("x_41_end_0"), val = tensor<int32, [4]>([512, 1, 32, 32])];
+            tensor<bool, [4]> x_41_end_mask_0 = const()[name = string("x_41_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 1, 32, 32]> x_41_cast_fp16 = slice_by_index(begin = x_41_begin_0, end = x_41_end_0, end_mask = x_41_end_mask_0, x = var_1115_cast_fp16)[name = string("x_41_cast_fp16")];
+            tensor<int32, [5]> var_1117 = const()[name = string("op_1117"), val = tensor<int32, [5]>([512, 1, 2, 32, 32])];
+            tensor<fp16, [512, 1, 2, 32, 32]> pair_27_cast_fp16 = reshape(shape = var_1117, x = inv_27_cast_fp16)[name = string("pair_27_cast_fp16")];
+            tensor<int32, [5]> var_1119_begin_0 = const()[name = string("op_1119_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_1119_end_0 = const()[name = string("op_1119_end_0"), val = tensor<int32, [5]>([512, 1, 1, 32, 32])];
+            tensor<bool, [5]> var_1119_end_mask_0 = const()[name = string("op_1119_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_1119_squeeze_mask_0 = const()[name = string("op_1119_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 1, 32, 32]> var_1119_cast_fp16 = slice_by_index(begin = var_1119_begin_0, end = var_1119_end_0, end_mask = var_1119_end_mask_0, squeeze_mask = var_1119_squeeze_mask_0, x = pair_27_cast_fp16)[name = string("op_1119_cast_fp16")];
+            tensor<int32, [5]> var_1122_begin_0 = const()[name = string("op_1122_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_1122_end_0 = const()[name = string("op_1122_end_0"), val = tensor<int32, [5]>([512, 1, 2, 32, 32])];
+            tensor<bool, [5]> var_1122_end_mask_0 = const()[name = string("op_1122_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_1122_squeeze_mask_0 = const()[name = string("op_1122_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 1, 32, 32]> var_1122_cast_fp16 = slice_by_index(begin = var_1122_begin_0, end = var_1122_end_0, end_mask = var_1122_end_mask_0, squeeze_mask = var_1122_squeeze_mask_0, x = pair_27_cast_fp16)[name = string("op_1122_cast_fp16")];
+            bool var_1125_transpose_x_0 = const()[name = string("op_1125_transpose_x_0"), val = bool(false)];
+            bool var_1125_transpose_y_0 = const()[name = string("op_1125_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 32, 32]> var_1125_cast_fp16 = matmul(transpose_x = var_1125_transpose_x_0, transpose_y = var_1125_transpose_y_0, x = x_41_cast_fp16, y = var_1119_cast_fp16)[name = string("op_1125_cast_fp16")];
+            bool var_1126_transpose_x_0 = const()[name = string("op_1126_transpose_x_0"), val = bool(false)];
+            bool var_1126_transpose_y_0 = const()[name = string("op_1126_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 32, 32]> var_1126_cast_fp16 = matmul(transpose_x = var_1126_transpose_x_0, transpose_y = var_1126_transpose_y_0, x = var_1122_cast_fp16, y = var_1125_cast_fp16)[name = string("op_1126_cast_fp16")];
+            fp16 const_23_promoted_to_fp16 = const()[name = string("const_23_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 1, 32, 32]> lower_23_cast_fp16 = mul(x = var_1126_cast_fp16, y = const_23_promoted_to_fp16)[name = string("lower_23_cast_fp16")];
+            bool top_23_interleave_0 = const()[name = string("top_23_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 32, 64]> top_23_cast_fp16 = concat(axis = var_802, interleave = top_23_interleave_0, values = (var_1119_cast_fp16, var_375_to_fp16))[name = string("top_23_cast_fp16")];
+            bool bottom_23_interleave_0 = const()[name = string("bottom_23_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 32, 64]> bottom_23_cast_fp16 = concat(axis = var_802, interleave = bottom_23_interleave_0, values = (lower_23_cast_fp16, var_1122_cast_fp16))[name = string("bottom_23_cast_fp16")];
+            bool inv_29_interleave_0 = const()[name = string("inv_29_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 64, 64]> inv_29_cast_fp16 = concat(axis = var_787, interleave = inv_29_interleave_0, values = (top_23_cast_fp16, bottom_23_cast_fp16))[name = string("inv_29_cast_fp16")];
+            tensor<int32, [4]> var_1135 = const()[name = string("op_1135"), val = tensor<int32, [4]>([32, 16, 64, 64])];
+            tensor<fp16, [32, 16, 64, 64]> inv_31_cast_fp16 = reshape(shape = var_1135, x = inv_29_cast_fp16)[name = string("inv_31_cast_fp16")];
+            bool new_v_3_transpose_x_0 = const()[name = string("new_v_3_transpose_x_0"), val = bool(false)];
+            bool new_v_3_transpose_y_0 = const()[name = string("new_v_3_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 16, 64, 128]> new_v_3_cast_fp16 = matmul(transpose_x = new_v_3_transpose_x_0, transpose_y = new_v_3_transpose_y_0, x = inv_31_cast_fp16, y = v_beta_3_cast_fp16)[name = string("new_v_3_cast_fp16")];
+            tensor<fp16, [32, 16, 64]> var_1138_cast_fp16 = exp(x = cum_3_cast_fp16)[name = string("op_1138_cast_fp16")];
+            tensor<int32, [1]> var_1139_axes_0 = const()[name = string("op_1139_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 16, 64, 1]> var_1139_cast_fp16 = expand_dims(axes = var_1139_axes_0, x = var_1138_cast_fp16)[name = string("op_1139_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 128]> var_1140_cast_fp16 = mul(x = k_beta_3_cast_fp16, y = var_1139_cast_fp16)[name = string("op_1140_cast_fp16")];
+            bool k_cumdecay_3_transpose_x_0 = const()[name = string("k_cumdecay_3_transpose_x_0"), val = bool(false)];
+            bool k_cumdecay_3_transpose_y_0 = const()[name = string("k_cumdecay_3_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 16, 64, 128]> k_cumdecay_3_cast_fp16 = matmul(transpose_x = k_cumdecay_3_transpose_x_0, transpose_y = k_cumdecay_3_transpose_y_0, x = inv_31_cast_fp16, y = var_1140_cast_fp16)[name = string("k_cumdecay_3_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 128]> q_dec_3_cast_fp16 = mul(x = q_19_cast_fp16, y = var_1139_cast_fp16)[name = string("q_dec_3_cast_fp16")];
+            tensor<fp16, [32, 16, 64]> var_1145_cast_fp16 = exp(x = to_end_3_cast_fp16)[name = string("op_1145_cast_fp16")];
+            tensor<int32, [1]> var_1146_axes_0 = const()[name = string("op_1146_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 16, 64, 1]> var_1146_cast_fp16 = expand_dims(axes = var_1146_axes_0, x = var_1145_cast_fp16)[name = string("op_1146_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 128]> k_dec_3_cast_fp16 = mul(x = k_15_cast_fp16, y = var_1146_cast_fp16)[name = string("k_dec_3_cast_fp16")];
+            tensor<int32, [3]> var_1148_begin_0 = const()[name = string("op_1148_begin_0"), val = tensor<int32, [3]>([0, 0, -1])];
+            tensor<int32, [3]> var_1148_end_0 = const()[name = string("op_1148_end_0"), val = tensor<int32, [3]>([32, 16, 64])];
+            tensor<bool, [3]> var_1148_end_mask_0 = const()[name = string("op_1148_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
+            tensor<fp16, [32, 16, 1]> var_1148_cast_fp16 = slice_by_index(begin = var_1148_begin_0, end = var_1148_end_0, end_mask = var_1148_end_mask_0, x = cum_3_cast_fp16)[name = string("op_1148_cast_fp16")];
+            tensor<fp16, [32, 16, 1]> var_1149_cast_fp16 = exp(x = var_1148_cast_fp16)[name = string("op_1149_cast_fp16")];
+            tensor<int32, [1]> chunk_decay_3_axes_0 = const()[name = string("chunk_decay_3_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 16, 1, 1]> chunk_decay_3_cast_fp16 = expand_dims(axes = chunk_decay_3_axes_0, x = var_1149_cast_fp16)[name = string("chunk_decay_3_cast_fp16")];
+            tensor<int32, [4]> var_1154_begin_0 = const()[name = string("op_1154_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_1154_end_0 = const()[name = string("op_1154_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_1154_end_mask_0 = const()[name = string("op_1154_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1154_squeeze_mask_0 = const()[name = string("op_1154_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1154_cast_fp16 = slice_by_index(begin = var_1154_begin_0, end = var_1154_end_0, end_mask = var_1154_end_mask_0, squeeze_mask = var_1154_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1154_cast_fp16")];
+            tensor<int32, [4]> var_1156_begin_0 = const()[name = string("op_1156_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_1156_end_0 = const()[name = string("op_1156_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_1156_end_mask_0 = const()[name = string("op_1156_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1156_squeeze_mask_0 = const()[name = string("op_1156_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1156_cast_fp16 = slice_by_index(begin = var_1156_begin_0, end = var_1156_end_0, end_mask = var_1156_end_mask_0, squeeze_mask = var_1156_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1156_cast_fp16")];
+            bool var_1157_transpose_x_0 = const()[name = string("op_1157_transpose_x_0"), val = bool(false)];
+            bool var_1157_transpose_y_0 = const()[name = string("op_1157_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1157_cast_fp16 = matmul(transpose_x = var_1157_transpose_x_0, transpose_y = var_1157_transpose_y_0, x = var_1156_cast_fp16, y = state_1_to_fp16)[name = string("op_1157_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_33_cast_fp16 = sub(x = var_1154_cast_fp16, y = var_1157_cast_fp16)[name = string("v_new_33_cast_fp16")];
+            tensor<int32, [4]> var_1160_begin_0 = const()[name = string("op_1160_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_1160_end_0 = const()[name = string("op_1160_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_1160_end_mask_0 = const()[name = string("op_1160_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1160_squeeze_mask_0 = const()[name = string("op_1160_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1160_cast_fp16 = slice_by_index(begin = var_1160_begin_0, end = var_1160_end_0, end_mask = var_1160_end_mask_0, squeeze_mask = var_1160_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1160_cast_fp16")];
+            bool var_1161_transpose_x_0 = const()[name = string("op_1161_transpose_x_0"), val = bool(false)];
+            bool var_1161_transpose_y_0 = const()[name = string("op_1161_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1161_cast_fp16 = matmul(transpose_x = var_1161_transpose_x_0, transpose_y = var_1161_transpose_y_0, x = var_1160_cast_fp16, y = state_1_to_fp16)[name = string("op_1161_cast_fp16")];
+            tensor<int32, [4]> var_1163_begin_0 = const()[name = string("op_1163_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_1163_end_0 = const()[name = string("op_1163_end_0"), val = tensor<int32, [4]>([32, 1, 64, 64])];
+            tensor<bool, [4]> var_1163_end_mask_0 = const()[name = string("op_1163_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1163_squeeze_mask_0 = const()[name = string("op_1163_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1163_cast_fp16 = slice_by_index(begin = var_1163_begin_0, end = var_1163_end_0, end_mask = var_1163_end_mask_0, squeeze_mask = var_1163_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1163_cast_fp16")];
+            bool var_1164_transpose_x_0 = const()[name = string("op_1164_transpose_x_0"), val = bool(false)];
+            bool var_1164_transpose_y_0 = const()[name = string("op_1164_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1164_cast_fp16 = matmul(transpose_x = var_1164_transpose_x_0, transpose_y = var_1164_transpose_y_0, x = var_1163_cast_fp16, y = v_new_33_cast_fp16)[name = string("op_1164_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1165_cast_fp16 = add(x = var_1161_cast_fp16, y = var_1164_cast_fp16)[name = string("op_1165_cast_fp16")];
+            tensor<int32, [4]> var_1167_begin_0 = const()[name = string("op_1167_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_1167_end_0 = const()[name = string("op_1167_end_0"), val = tensor<int32, [4]>([32, 1, 1, 1])];
+            tensor<bool, [4]> var_1167_end_mask_0 = const()[name = string("op_1167_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1167_squeeze_mask_0 = const()[name = string("op_1167_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1167_cast_fp16 = slice_by_index(begin = var_1167_begin_0, end = var_1167_end_0, end_mask = var_1167_end_mask_0, squeeze_mask = var_1167_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1167_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1168_cast_fp16 = mul(x = state_1_to_fp16, y = var_1167_cast_fp16)[name = string("op_1168_cast_fp16")];
+            tensor<int32, [4]> var_1170_begin_0 = const()[name = string("op_1170_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_1170_end_0 = const()[name = string("op_1170_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_1170_end_mask_0 = const()[name = string("op_1170_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1170_squeeze_mask_0 = const()[name = string("op_1170_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1170_cast_fp16 = slice_by_index(begin = var_1170_begin_0, end = var_1170_end_0, end_mask = var_1170_end_mask_0, squeeze_mask = var_1170_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1170_cast_fp16")];
+            bool var_1172_transpose_x_1 = const()[name = string("op_1172_transpose_x_1"), val = bool(true)];
+            bool var_1172_transpose_y_1 = const()[name = string("op_1172_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1172_cast_fp16 = matmul(transpose_x = var_1172_transpose_x_1, transpose_y = var_1172_transpose_y_1, x = var_1170_cast_fp16, y = v_new_33_cast_fp16)[name = string("op_1172_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_35_cast_fp16 = add(x = var_1168_cast_fp16, y = var_1172_cast_fp16)[name = string("state_35_cast_fp16")];
+            tensor<int32, [4]> var_1175_begin_0 = const()[name = string("op_1175_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_1175_end_0 = const()[name = string("op_1175_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_1175_end_mask_0 = const()[name = string("op_1175_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1175_squeeze_mask_0 = const()[name = string("op_1175_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1175_cast_fp16 = slice_by_index(begin = var_1175_begin_0, end = var_1175_end_0, end_mask = var_1175_end_mask_0, squeeze_mask = var_1175_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1175_cast_fp16")];
+            tensor<int32, [4]> var_1177_begin_0 = const()[name = string("op_1177_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_1177_end_0 = const()[name = string("op_1177_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_1177_end_mask_0 = const()[name = string("op_1177_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1177_squeeze_mask_0 = const()[name = string("op_1177_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1177_cast_fp16 = slice_by_index(begin = var_1177_begin_0, end = var_1177_end_0, end_mask = var_1177_end_mask_0, squeeze_mask = var_1177_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1177_cast_fp16")];
+            bool var_1178_transpose_x_0 = const()[name = string("op_1178_transpose_x_0"), val = bool(false)];
+            bool var_1178_transpose_y_0 = const()[name = string("op_1178_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1178_cast_fp16 = matmul(transpose_x = var_1178_transpose_x_0, transpose_y = var_1178_transpose_y_0, x = var_1177_cast_fp16, y = state_35_cast_fp16)[name = string("op_1178_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_35_cast_fp16 = sub(x = var_1175_cast_fp16, y = var_1178_cast_fp16)[name = string("v_new_35_cast_fp16")];
+            tensor<int32, [4]> var_1181_begin_0 = const()[name = string("op_1181_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_1181_end_0 = const()[name = string("op_1181_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_1181_end_mask_0 = const()[name = string("op_1181_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1181_squeeze_mask_0 = const()[name = string("op_1181_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1181_cast_fp16 = slice_by_index(begin = var_1181_begin_0, end = var_1181_end_0, end_mask = var_1181_end_mask_0, squeeze_mask = var_1181_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1181_cast_fp16")];
+            bool var_1182_transpose_x_0 = const()[name = string("op_1182_transpose_x_0"), val = bool(false)];
+            bool var_1182_transpose_y_0 = const()[name = string("op_1182_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1182_cast_fp16 = matmul(transpose_x = var_1182_transpose_x_0, transpose_y = var_1182_transpose_y_0, x = var_1181_cast_fp16, y = state_35_cast_fp16)[name = string("op_1182_cast_fp16")];
+            tensor<int32, [4]> var_1184_begin_0 = const()[name = string("op_1184_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_1184_end_0 = const()[name = string("op_1184_end_0"), val = tensor<int32, [4]>([32, 2, 64, 64])];
+            tensor<bool, [4]> var_1184_end_mask_0 = const()[name = string("op_1184_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1184_squeeze_mask_0 = const()[name = string("op_1184_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1184_cast_fp16 = slice_by_index(begin = var_1184_begin_0, end = var_1184_end_0, end_mask = var_1184_end_mask_0, squeeze_mask = var_1184_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1184_cast_fp16")];
+            bool var_1185_transpose_x_0 = const()[name = string("op_1185_transpose_x_0"), val = bool(false)];
+            bool var_1185_transpose_y_0 = const()[name = string("op_1185_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1185_cast_fp16 = matmul(transpose_x = var_1185_transpose_x_0, transpose_y = var_1185_transpose_y_0, x = var_1184_cast_fp16, y = v_new_35_cast_fp16)[name = string("op_1185_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1186_cast_fp16 = add(x = var_1182_cast_fp16, y = var_1185_cast_fp16)[name = string("op_1186_cast_fp16")];
+            tensor<int32, [4]> var_1188_begin_0 = const()[name = string("op_1188_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_1188_end_0 = const()[name = string("op_1188_end_0"), val = tensor<int32, [4]>([32, 2, 1, 1])];
+            tensor<bool, [4]> var_1188_end_mask_0 = const()[name = string("op_1188_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1188_squeeze_mask_0 = const()[name = string("op_1188_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1188_cast_fp16 = slice_by_index(begin = var_1188_begin_0, end = var_1188_end_0, end_mask = var_1188_end_mask_0, squeeze_mask = var_1188_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1188_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1189_cast_fp16 = mul(x = state_35_cast_fp16, y = var_1188_cast_fp16)[name = string("op_1189_cast_fp16")];
+            tensor<int32, [4]> var_1191_begin_0 = const()[name = string("op_1191_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_1191_end_0 = const()[name = string("op_1191_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_1191_end_mask_0 = const()[name = string("op_1191_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1191_squeeze_mask_0 = const()[name = string("op_1191_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1191_cast_fp16 = slice_by_index(begin = var_1191_begin_0, end = var_1191_end_0, end_mask = var_1191_end_mask_0, squeeze_mask = var_1191_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1191_cast_fp16")];
+            bool var_1193_transpose_x_1 = const()[name = string("op_1193_transpose_x_1"), val = bool(true)];
+            bool var_1193_transpose_y_1 = const()[name = string("op_1193_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1193_cast_fp16 = matmul(transpose_x = var_1193_transpose_x_1, transpose_y = var_1193_transpose_y_1, x = var_1191_cast_fp16, y = v_new_35_cast_fp16)[name = string("op_1193_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_37_cast_fp16 = add(x = var_1189_cast_fp16, y = var_1193_cast_fp16)[name = string("state_37_cast_fp16")];
+            tensor<int32, [4]> var_1196_begin_0 = const()[name = string("op_1196_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_1196_end_0 = const()[name = string("op_1196_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_1196_end_mask_0 = const()[name = string("op_1196_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1196_squeeze_mask_0 = const()[name = string("op_1196_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1196_cast_fp16 = slice_by_index(begin = var_1196_begin_0, end = var_1196_end_0, end_mask = var_1196_end_mask_0, squeeze_mask = var_1196_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1196_cast_fp16")];
+            tensor<int32, [4]> var_1198_begin_0 = const()[name = string("op_1198_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_1198_end_0 = const()[name = string("op_1198_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_1198_end_mask_0 = const()[name = string("op_1198_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1198_squeeze_mask_0 = const()[name = string("op_1198_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1198_cast_fp16 = slice_by_index(begin = var_1198_begin_0, end = var_1198_end_0, end_mask = var_1198_end_mask_0, squeeze_mask = var_1198_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1198_cast_fp16")];
+            bool var_1199_transpose_x_0 = const()[name = string("op_1199_transpose_x_0"), val = bool(false)];
+            bool var_1199_transpose_y_0 = const()[name = string("op_1199_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1199_cast_fp16 = matmul(transpose_x = var_1199_transpose_x_0, transpose_y = var_1199_transpose_y_0, x = var_1198_cast_fp16, y = state_37_cast_fp16)[name = string("op_1199_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_37_cast_fp16 = sub(x = var_1196_cast_fp16, y = var_1199_cast_fp16)[name = string("v_new_37_cast_fp16")];
+            tensor<int32, [4]> var_1202_begin_0 = const()[name = string("op_1202_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_1202_end_0 = const()[name = string("op_1202_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_1202_end_mask_0 = const()[name = string("op_1202_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1202_squeeze_mask_0 = const()[name = string("op_1202_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1202_cast_fp16 = slice_by_index(begin = var_1202_begin_0, end = var_1202_end_0, end_mask = var_1202_end_mask_0, squeeze_mask = var_1202_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1202_cast_fp16")];
+            bool var_1203_transpose_x_0 = const()[name = string("op_1203_transpose_x_0"), val = bool(false)];
+            bool var_1203_transpose_y_0 = const()[name = string("op_1203_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1203_cast_fp16 = matmul(transpose_x = var_1203_transpose_x_0, transpose_y = var_1203_transpose_y_0, x = var_1202_cast_fp16, y = state_37_cast_fp16)[name = string("op_1203_cast_fp16")];
+            tensor<int32, [4]> var_1205_begin_0 = const()[name = string("op_1205_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_1205_end_0 = const()[name = string("op_1205_end_0"), val = tensor<int32, [4]>([32, 3, 64, 64])];
+            tensor<bool, [4]> var_1205_end_mask_0 = const()[name = string("op_1205_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1205_squeeze_mask_0 = const()[name = string("op_1205_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1205_cast_fp16 = slice_by_index(begin = var_1205_begin_0, end = var_1205_end_0, end_mask = var_1205_end_mask_0, squeeze_mask = var_1205_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1205_cast_fp16")];
+            bool var_1206_transpose_x_0 = const()[name = string("op_1206_transpose_x_0"), val = bool(false)];
+            bool var_1206_transpose_y_0 = const()[name = string("op_1206_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1206_cast_fp16 = matmul(transpose_x = var_1206_transpose_x_0, transpose_y = var_1206_transpose_y_0, x = var_1205_cast_fp16, y = v_new_37_cast_fp16)[name = string("op_1206_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1207_cast_fp16 = add(x = var_1203_cast_fp16, y = var_1206_cast_fp16)[name = string("op_1207_cast_fp16")];
+            tensor<int32, [4]> var_1209_begin_0 = const()[name = string("op_1209_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_1209_end_0 = const()[name = string("op_1209_end_0"), val = tensor<int32, [4]>([32, 3, 1, 1])];
+            tensor<bool, [4]> var_1209_end_mask_0 = const()[name = string("op_1209_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1209_squeeze_mask_0 = const()[name = string("op_1209_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1209_cast_fp16 = slice_by_index(begin = var_1209_begin_0, end = var_1209_end_0, end_mask = var_1209_end_mask_0, squeeze_mask = var_1209_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1209_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1210_cast_fp16 = mul(x = state_37_cast_fp16, y = var_1209_cast_fp16)[name = string("op_1210_cast_fp16")];
+            tensor<int32, [4]> var_1212_begin_0 = const()[name = string("op_1212_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_1212_end_0 = const()[name = string("op_1212_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_1212_end_mask_0 = const()[name = string("op_1212_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1212_squeeze_mask_0 = const()[name = string("op_1212_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1212_cast_fp16 = slice_by_index(begin = var_1212_begin_0, end = var_1212_end_0, end_mask = var_1212_end_mask_0, squeeze_mask = var_1212_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1212_cast_fp16")];
+            bool var_1214_transpose_x_1 = const()[name = string("op_1214_transpose_x_1"), val = bool(true)];
+            bool var_1214_transpose_y_1 = const()[name = string("op_1214_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1214_cast_fp16 = matmul(transpose_x = var_1214_transpose_x_1, transpose_y = var_1214_transpose_y_1, x = var_1212_cast_fp16, y = v_new_37_cast_fp16)[name = string("op_1214_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_39_cast_fp16 = add(x = var_1210_cast_fp16, y = var_1214_cast_fp16)[name = string("state_39_cast_fp16")];
+            tensor<int32, [4]> var_1217_begin_0 = const()[name = string("op_1217_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_1217_end_0 = const()[name = string("op_1217_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_1217_end_mask_0 = const()[name = string("op_1217_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1217_squeeze_mask_0 = const()[name = string("op_1217_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1217_cast_fp16 = slice_by_index(begin = var_1217_begin_0, end = var_1217_end_0, end_mask = var_1217_end_mask_0, squeeze_mask = var_1217_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1217_cast_fp16")];
+            tensor<int32, [4]> var_1219_begin_0 = const()[name = string("op_1219_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_1219_end_0 = const()[name = string("op_1219_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_1219_end_mask_0 = const()[name = string("op_1219_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1219_squeeze_mask_0 = const()[name = string("op_1219_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1219_cast_fp16 = slice_by_index(begin = var_1219_begin_0, end = var_1219_end_0, end_mask = var_1219_end_mask_0, squeeze_mask = var_1219_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1219_cast_fp16")];
+            bool var_1220_transpose_x_0 = const()[name = string("op_1220_transpose_x_0"), val = bool(false)];
+            bool var_1220_transpose_y_0 = const()[name = string("op_1220_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1220_cast_fp16 = matmul(transpose_x = var_1220_transpose_x_0, transpose_y = var_1220_transpose_y_0, x = var_1219_cast_fp16, y = state_39_cast_fp16)[name = string("op_1220_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_39_cast_fp16 = sub(x = var_1217_cast_fp16, y = var_1220_cast_fp16)[name = string("v_new_39_cast_fp16")];
+            tensor<int32, [4]> var_1223_begin_0 = const()[name = string("op_1223_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_1223_end_0 = const()[name = string("op_1223_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_1223_end_mask_0 = const()[name = string("op_1223_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1223_squeeze_mask_0 = const()[name = string("op_1223_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1223_cast_fp16 = slice_by_index(begin = var_1223_begin_0, end = var_1223_end_0, end_mask = var_1223_end_mask_0, squeeze_mask = var_1223_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1223_cast_fp16")];
+            bool var_1224_transpose_x_0 = const()[name = string("op_1224_transpose_x_0"), val = bool(false)];
+            bool var_1224_transpose_y_0 = const()[name = string("op_1224_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1224_cast_fp16 = matmul(transpose_x = var_1224_transpose_x_0, transpose_y = var_1224_transpose_y_0, x = var_1223_cast_fp16, y = state_39_cast_fp16)[name = string("op_1224_cast_fp16")];
+            tensor<int32, [4]> var_1226_begin_0 = const()[name = string("op_1226_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_1226_end_0 = const()[name = string("op_1226_end_0"), val = tensor<int32, [4]>([32, 4, 64, 64])];
+            tensor<bool, [4]> var_1226_end_mask_0 = const()[name = string("op_1226_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1226_squeeze_mask_0 = const()[name = string("op_1226_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1226_cast_fp16 = slice_by_index(begin = var_1226_begin_0, end = var_1226_end_0, end_mask = var_1226_end_mask_0, squeeze_mask = var_1226_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1226_cast_fp16")];
+            bool var_1227_transpose_x_0 = const()[name = string("op_1227_transpose_x_0"), val = bool(false)];
+            bool var_1227_transpose_y_0 = const()[name = string("op_1227_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1227_cast_fp16 = matmul(transpose_x = var_1227_transpose_x_0, transpose_y = var_1227_transpose_y_0, x = var_1226_cast_fp16, y = v_new_39_cast_fp16)[name = string("op_1227_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1228_cast_fp16 = add(x = var_1224_cast_fp16, y = var_1227_cast_fp16)[name = string("op_1228_cast_fp16")];
+            tensor<int32, [4]> var_1230_begin_0 = const()[name = string("op_1230_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_1230_end_0 = const()[name = string("op_1230_end_0"), val = tensor<int32, [4]>([32, 4, 1, 1])];
+            tensor<bool, [4]> var_1230_end_mask_0 = const()[name = string("op_1230_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1230_squeeze_mask_0 = const()[name = string("op_1230_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1230_cast_fp16 = slice_by_index(begin = var_1230_begin_0, end = var_1230_end_0, end_mask = var_1230_end_mask_0, squeeze_mask = var_1230_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1230_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1231_cast_fp16 = mul(x = state_39_cast_fp16, y = var_1230_cast_fp16)[name = string("op_1231_cast_fp16")];
+            tensor<int32, [4]> var_1233_begin_0 = const()[name = string("op_1233_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_1233_end_0 = const()[name = string("op_1233_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_1233_end_mask_0 = const()[name = string("op_1233_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1233_squeeze_mask_0 = const()[name = string("op_1233_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1233_cast_fp16 = slice_by_index(begin = var_1233_begin_0, end = var_1233_end_0, end_mask = var_1233_end_mask_0, squeeze_mask = var_1233_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1233_cast_fp16")];
+            bool var_1235_transpose_x_1 = const()[name = string("op_1235_transpose_x_1"), val = bool(true)];
+            bool var_1235_transpose_y_1 = const()[name = string("op_1235_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1235_cast_fp16 = matmul(transpose_x = var_1235_transpose_x_1, transpose_y = var_1235_transpose_y_1, x = var_1233_cast_fp16, y = v_new_39_cast_fp16)[name = string("op_1235_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_41_cast_fp16 = add(x = var_1231_cast_fp16, y = var_1235_cast_fp16)[name = string("state_41_cast_fp16")];
+            tensor<int32, [4]> var_1238_begin_0 = const()[name = string("op_1238_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_1238_end_0 = const()[name = string("op_1238_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_1238_end_mask_0 = const()[name = string("op_1238_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1238_squeeze_mask_0 = const()[name = string("op_1238_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1238_cast_fp16 = slice_by_index(begin = var_1238_begin_0, end = var_1238_end_0, end_mask = var_1238_end_mask_0, squeeze_mask = var_1238_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1238_cast_fp16")];
+            tensor<int32, [4]> var_1240_begin_0 = const()[name = string("op_1240_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_1240_end_0 = const()[name = string("op_1240_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_1240_end_mask_0 = const()[name = string("op_1240_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1240_squeeze_mask_0 = const()[name = string("op_1240_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1240_cast_fp16 = slice_by_index(begin = var_1240_begin_0, end = var_1240_end_0, end_mask = var_1240_end_mask_0, squeeze_mask = var_1240_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1240_cast_fp16")];
+            bool var_1241_transpose_x_0 = const()[name = string("op_1241_transpose_x_0"), val = bool(false)];
+            bool var_1241_transpose_y_0 = const()[name = string("op_1241_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1241_cast_fp16 = matmul(transpose_x = var_1241_transpose_x_0, transpose_y = var_1241_transpose_y_0, x = var_1240_cast_fp16, y = state_41_cast_fp16)[name = string("op_1241_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_41_cast_fp16 = sub(x = var_1238_cast_fp16, y = var_1241_cast_fp16)[name = string("v_new_41_cast_fp16")];
+            tensor<int32, [4]> var_1244_begin_0 = const()[name = string("op_1244_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_1244_end_0 = const()[name = string("op_1244_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_1244_end_mask_0 = const()[name = string("op_1244_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1244_squeeze_mask_0 = const()[name = string("op_1244_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1244_cast_fp16 = slice_by_index(begin = var_1244_begin_0, end = var_1244_end_0, end_mask = var_1244_end_mask_0, squeeze_mask = var_1244_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1244_cast_fp16")];
+            bool var_1245_transpose_x_0 = const()[name = string("op_1245_transpose_x_0"), val = bool(false)];
+            bool var_1245_transpose_y_0 = const()[name = string("op_1245_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1245_cast_fp16 = matmul(transpose_x = var_1245_transpose_x_0, transpose_y = var_1245_transpose_y_0, x = var_1244_cast_fp16, y = state_41_cast_fp16)[name = string("op_1245_cast_fp16")];
+            tensor<int32, [4]> var_1247_begin_0 = const()[name = string("op_1247_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_1247_end_0 = const()[name = string("op_1247_end_0"), val = tensor<int32, [4]>([32, 5, 64, 64])];
+            tensor<bool, [4]> var_1247_end_mask_0 = const()[name = string("op_1247_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1247_squeeze_mask_0 = const()[name = string("op_1247_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1247_cast_fp16 = slice_by_index(begin = var_1247_begin_0, end = var_1247_end_0, end_mask = var_1247_end_mask_0, squeeze_mask = var_1247_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1247_cast_fp16")];
+            bool var_1248_transpose_x_0 = const()[name = string("op_1248_transpose_x_0"), val = bool(false)];
+            bool var_1248_transpose_y_0 = const()[name = string("op_1248_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1248_cast_fp16 = matmul(transpose_x = var_1248_transpose_x_0, transpose_y = var_1248_transpose_y_0, x = var_1247_cast_fp16, y = v_new_41_cast_fp16)[name = string("op_1248_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1249_cast_fp16 = add(x = var_1245_cast_fp16, y = var_1248_cast_fp16)[name = string("op_1249_cast_fp16")];
+            tensor<int32, [4]> var_1251_begin_0 = const()[name = string("op_1251_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_1251_end_0 = const()[name = string("op_1251_end_0"), val = tensor<int32, [4]>([32, 5, 1, 1])];
+            tensor<bool, [4]> var_1251_end_mask_0 = const()[name = string("op_1251_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1251_squeeze_mask_0 = const()[name = string("op_1251_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1251_cast_fp16 = slice_by_index(begin = var_1251_begin_0, end = var_1251_end_0, end_mask = var_1251_end_mask_0, squeeze_mask = var_1251_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1251_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1252_cast_fp16 = mul(x = state_41_cast_fp16, y = var_1251_cast_fp16)[name = string("op_1252_cast_fp16")];
+            tensor<int32, [4]> var_1254_begin_0 = const()[name = string("op_1254_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_1254_end_0 = const()[name = string("op_1254_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_1254_end_mask_0 = const()[name = string("op_1254_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1254_squeeze_mask_0 = const()[name = string("op_1254_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1254_cast_fp16 = slice_by_index(begin = var_1254_begin_0, end = var_1254_end_0, end_mask = var_1254_end_mask_0, squeeze_mask = var_1254_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1254_cast_fp16")];
+            bool var_1256_transpose_x_1 = const()[name = string("op_1256_transpose_x_1"), val = bool(true)];
+            bool var_1256_transpose_y_1 = const()[name = string("op_1256_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1256_cast_fp16 = matmul(transpose_x = var_1256_transpose_x_1, transpose_y = var_1256_transpose_y_1, x = var_1254_cast_fp16, y = v_new_41_cast_fp16)[name = string("op_1256_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_43_cast_fp16 = add(x = var_1252_cast_fp16, y = var_1256_cast_fp16)[name = string("state_43_cast_fp16")];
+            tensor<int32, [4]> var_1259_begin_0 = const()[name = string("op_1259_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_1259_end_0 = const()[name = string("op_1259_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_1259_end_mask_0 = const()[name = string("op_1259_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1259_squeeze_mask_0 = const()[name = string("op_1259_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1259_cast_fp16 = slice_by_index(begin = var_1259_begin_0, end = var_1259_end_0, end_mask = var_1259_end_mask_0, squeeze_mask = var_1259_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1259_cast_fp16")];
+            tensor<int32, [4]> var_1261_begin_0 = const()[name = string("op_1261_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_1261_end_0 = const()[name = string("op_1261_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_1261_end_mask_0 = const()[name = string("op_1261_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1261_squeeze_mask_0 = const()[name = string("op_1261_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1261_cast_fp16 = slice_by_index(begin = var_1261_begin_0, end = var_1261_end_0, end_mask = var_1261_end_mask_0, squeeze_mask = var_1261_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1261_cast_fp16")];
+            bool var_1262_transpose_x_0 = const()[name = string("op_1262_transpose_x_0"), val = bool(false)];
+            bool var_1262_transpose_y_0 = const()[name = string("op_1262_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1262_cast_fp16 = matmul(transpose_x = var_1262_transpose_x_0, transpose_y = var_1262_transpose_y_0, x = var_1261_cast_fp16, y = state_43_cast_fp16)[name = string("op_1262_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_43_cast_fp16 = sub(x = var_1259_cast_fp16, y = var_1262_cast_fp16)[name = string("v_new_43_cast_fp16")];
+            tensor<int32, [4]> var_1265_begin_0 = const()[name = string("op_1265_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_1265_end_0 = const()[name = string("op_1265_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_1265_end_mask_0 = const()[name = string("op_1265_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1265_squeeze_mask_0 = const()[name = string("op_1265_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1265_cast_fp16 = slice_by_index(begin = var_1265_begin_0, end = var_1265_end_0, end_mask = var_1265_end_mask_0, squeeze_mask = var_1265_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1265_cast_fp16")];
+            bool var_1266_transpose_x_0 = const()[name = string("op_1266_transpose_x_0"), val = bool(false)];
+            bool var_1266_transpose_y_0 = const()[name = string("op_1266_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1266_cast_fp16 = matmul(transpose_x = var_1266_transpose_x_0, transpose_y = var_1266_transpose_y_0, x = var_1265_cast_fp16, y = state_43_cast_fp16)[name = string("op_1266_cast_fp16")];
+            tensor<int32, [4]> var_1268_begin_0 = const()[name = string("op_1268_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_1268_end_0 = const()[name = string("op_1268_end_0"), val = tensor<int32, [4]>([32, 6, 64, 64])];
+            tensor<bool, [4]> var_1268_end_mask_0 = const()[name = string("op_1268_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1268_squeeze_mask_0 = const()[name = string("op_1268_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1268_cast_fp16 = slice_by_index(begin = var_1268_begin_0, end = var_1268_end_0, end_mask = var_1268_end_mask_0, squeeze_mask = var_1268_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1268_cast_fp16")];
+            bool var_1269_transpose_x_0 = const()[name = string("op_1269_transpose_x_0"), val = bool(false)];
+            bool var_1269_transpose_y_0 = const()[name = string("op_1269_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1269_cast_fp16 = matmul(transpose_x = var_1269_transpose_x_0, transpose_y = var_1269_transpose_y_0, x = var_1268_cast_fp16, y = v_new_43_cast_fp16)[name = string("op_1269_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1270_cast_fp16 = add(x = var_1266_cast_fp16, y = var_1269_cast_fp16)[name = string("op_1270_cast_fp16")];
+            tensor<int32, [4]> var_1272_begin_0 = const()[name = string("op_1272_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_1272_end_0 = const()[name = string("op_1272_end_0"), val = tensor<int32, [4]>([32, 6, 1, 1])];
+            tensor<bool, [4]> var_1272_end_mask_0 = const()[name = string("op_1272_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1272_squeeze_mask_0 = const()[name = string("op_1272_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1272_cast_fp16 = slice_by_index(begin = var_1272_begin_0, end = var_1272_end_0, end_mask = var_1272_end_mask_0, squeeze_mask = var_1272_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1272_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1273_cast_fp16 = mul(x = state_43_cast_fp16, y = var_1272_cast_fp16)[name = string("op_1273_cast_fp16")];
+            tensor<int32, [4]> var_1275_begin_0 = const()[name = string("op_1275_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_1275_end_0 = const()[name = string("op_1275_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_1275_end_mask_0 = const()[name = string("op_1275_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1275_squeeze_mask_0 = const()[name = string("op_1275_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1275_cast_fp16 = slice_by_index(begin = var_1275_begin_0, end = var_1275_end_0, end_mask = var_1275_end_mask_0, squeeze_mask = var_1275_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1275_cast_fp16")];
+            bool var_1277_transpose_x_1 = const()[name = string("op_1277_transpose_x_1"), val = bool(true)];
+            bool var_1277_transpose_y_1 = const()[name = string("op_1277_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1277_cast_fp16 = matmul(transpose_x = var_1277_transpose_x_1, transpose_y = var_1277_transpose_y_1, x = var_1275_cast_fp16, y = v_new_43_cast_fp16)[name = string("op_1277_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_45_cast_fp16 = add(x = var_1273_cast_fp16, y = var_1277_cast_fp16)[name = string("state_45_cast_fp16")];
+            tensor<int32, [4]> var_1280_begin_0 = const()[name = string("op_1280_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_1280_end_0 = const()[name = string("op_1280_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_1280_end_mask_0 = const()[name = string("op_1280_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1280_squeeze_mask_0 = const()[name = string("op_1280_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1280_cast_fp16 = slice_by_index(begin = var_1280_begin_0, end = var_1280_end_0, end_mask = var_1280_end_mask_0, squeeze_mask = var_1280_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1280_cast_fp16")];
+            tensor<int32, [4]> var_1282_begin_0 = const()[name = string("op_1282_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_1282_end_0 = const()[name = string("op_1282_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_1282_end_mask_0 = const()[name = string("op_1282_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1282_squeeze_mask_0 = const()[name = string("op_1282_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1282_cast_fp16 = slice_by_index(begin = var_1282_begin_0, end = var_1282_end_0, end_mask = var_1282_end_mask_0, squeeze_mask = var_1282_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1282_cast_fp16")];
+            bool var_1283_transpose_x_0 = const()[name = string("op_1283_transpose_x_0"), val = bool(false)];
+            bool var_1283_transpose_y_0 = const()[name = string("op_1283_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1283_cast_fp16 = matmul(transpose_x = var_1283_transpose_x_0, transpose_y = var_1283_transpose_y_0, x = var_1282_cast_fp16, y = state_45_cast_fp16)[name = string("op_1283_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_45_cast_fp16 = sub(x = var_1280_cast_fp16, y = var_1283_cast_fp16)[name = string("v_new_45_cast_fp16")];
+            tensor<int32, [4]> var_1286_begin_0 = const()[name = string("op_1286_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_1286_end_0 = const()[name = string("op_1286_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_1286_end_mask_0 = const()[name = string("op_1286_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1286_squeeze_mask_0 = const()[name = string("op_1286_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1286_cast_fp16 = slice_by_index(begin = var_1286_begin_0, end = var_1286_end_0, end_mask = var_1286_end_mask_0, squeeze_mask = var_1286_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1286_cast_fp16")];
+            bool var_1287_transpose_x_0 = const()[name = string("op_1287_transpose_x_0"), val = bool(false)];
+            bool var_1287_transpose_y_0 = const()[name = string("op_1287_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1287_cast_fp16 = matmul(transpose_x = var_1287_transpose_x_0, transpose_y = var_1287_transpose_y_0, x = var_1286_cast_fp16, y = state_45_cast_fp16)[name = string("op_1287_cast_fp16")];
+            tensor<int32, [4]> var_1289_begin_0 = const()[name = string("op_1289_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_1289_end_0 = const()[name = string("op_1289_end_0"), val = tensor<int32, [4]>([32, 7, 64, 64])];
+            tensor<bool, [4]> var_1289_end_mask_0 = const()[name = string("op_1289_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1289_squeeze_mask_0 = const()[name = string("op_1289_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1289_cast_fp16 = slice_by_index(begin = var_1289_begin_0, end = var_1289_end_0, end_mask = var_1289_end_mask_0, squeeze_mask = var_1289_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1289_cast_fp16")];
+            bool var_1290_transpose_x_0 = const()[name = string("op_1290_transpose_x_0"), val = bool(false)];
+            bool var_1290_transpose_y_0 = const()[name = string("op_1290_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1290_cast_fp16 = matmul(transpose_x = var_1290_transpose_x_0, transpose_y = var_1290_transpose_y_0, x = var_1289_cast_fp16, y = v_new_45_cast_fp16)[name = string("op_1290_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1291_cast_fp16 = add(x = var_1287_cast_fp16, y = var_1290_cast_fp16)[name = string("op_1291_cast_fp16")];
+            tensor<int32, [4]> var_1293_begin_0 = const()[name = string("op_1293_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_1293_end_0 = const()[name = string("op_1293_end_0"), val = tensor<int32, [4]>([32, 7, 1, 1])];
+            tensor<bool, [4]> var_1293_end_mask_0 = const()[name = string("op_1293_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1293_squeeze_mask_0 = const()[name = string("op_1293_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1293_cast_fp16 = slice_by_index(begin = var_1293_begin_0, end = var_1293_end_0, end_mask = var_1293_end_mask_0, squeeze_mask = var_1293_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1293_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1294_cast_fp16 = mul(x = state_45_cast_fp16, y = var_1293_cast_fp16)[name = string("op_1294_cast_fp16")];
+            tensor<int32, [4]> var_1296_begin_0 = const()[name = string("op_1296_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_1296_end_0 = const()[name = string("op_1296_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_1296_end_mask_0 = const()[name = string("op_1296_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1296_squeeze_mask_0 = const()[name = string("op_1296_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1296_cast_fp16 = slice_by_index(begin = var_1296_begin_0, end = var_1296_end_0, end_mask = var_1296_end_mask_0, squeeze_mask = var_1296_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1296_cast_fp16")];
+            bool var_1298_transpose_x_1 = const()[name = string("op_1298_transpose_x_1"), val = bool(true)];
+            bool var_1298_transpose_y_1 = const()[name = string("op_1298_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1298_cast_fp16 = matmul(transpose_x = var_1298_transpose_x_1, transpose_y = var_1298_transpose_y_1, x = var_1296_cast_fp16, y = v_new_45_cast_fp16)[name = string("op_1298_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_47_cast_fp16 = add(x = var_1294_cast_fp16, y = var_1298_cast_fp16)[name = string("state_47_cast_fp16")];
+            tensor<int32, [4]> var_1301_begin_0 = const()[name = string("op_1301_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_1301_end_0 = const()[name = string("op_1301_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_1301_end_mask_0 = const()[name = string("op_1301_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1301_squeeze_mask_0 = const()[name = string("op_1301_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1301_cast_fp16 = slice_by_index(begin = var_1301_begin_0, end = var_1301_end_0, end_mask = var_1301_end_mask_0, squeeze_mask = var_1301_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1301_cast_fp16")];
+            tensor<int32, [4]> var_1303_begin_0 = const()[name = string("op_1303_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_1303_end_0 = const()[name = string("op_1303_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_1303_end_mask_0 = const()[name = string("op_1303_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1303_squeeze_mask_0 = const()[name = string("op_1303_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1303_cast_fp16 = slice_by_index(begin = var_1303_begin_0, end = var_1303_end_0, end_mask = var_1303_end_mask_0, squeeze_mask = var_1303_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1303_cast_fp16")];
+            bool var_1304_transpose_x_0 = const()[name = string("op_1304_transpose_x_0"), val = bool(false)];
+            bool var_1304_transpose_y_0 = const()[name = string("op_1304_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1304_cast_fp16 = matmul(transpose_x = var_1304_transpose_x_0, transpose_y = var_1304_transpose_y_0, x = var_1303_cast_fp16, y = state_47_cast_fp16)[name = string("op_1304_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_47_cast_fp16 = sub(x = var_1301_cast_fp16, y = var_1304_cast_fp16)[name = string("v_new_47_cast_fp16")];
+            tensor<int32, [4]> var_1307_begin_0 = const()[name = string("op_1307_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_1307_end_0 = const()[name = string("op_1307_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_1307_end_mask_0 = const()[name = string("op_1307_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1307_squeeze_mask_0 = const()[name = string("op_1307_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1307_cast_fp16 = slice_by_index(begin = var_1307_begin_0, end = var_1307_end_0, end_mask = var_1307_end_mask_0, squeeze_mask = var_1307_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1307_cast_fp16")];
+            bool var_1308_transpose_x_0 = const()[name = string("op_1308_transpose_x_0"), val = bool(false)];
+            bool var_1308_transpose_y_0 = const()[name = string("op_1308_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1308_cast_fp16 = matmul(transpose_x = var_1308_transpose_x_0, transpose_y = var_1308_transpose_y_0, x = var_1307_cast_fp16, y = state_47_cast_fp16)[name = string("op_1308_cast_fp16")];
+            tensor<int32, [4]> var_1310_begin_0 = const()[name = string("op_1310_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_1310_end_0 = const()[name = string("op_1310_end_0"), val = tensor<int32, [4]>([32, 8, 64, 64])];
+            tensor<bool, [4]> var_1310_end_mask_0 = const()[name = string("op_1310_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1310_squeeze_mask_0 = const()[name = string("op_1310_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1310_cast_fp16 = slice_by_index(begin = var_1310_begin_0, end = var_1310_end_0, end_mask = var_1310_end_mask_0, squeeze_mask = var_1310_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1310_cast_fp16")];
+            bool var_1311_transpose_x_0 = const()[name = string("op_1311_transpose_x_0"), val = bool(false)];
+            bool var_1311_transpose_y_0 = const()[name = string("op_1311_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1311_cast_fp16 = matmul(transpose_x = var_1311_transpose_x_0, transpose_y = var_1311_transpose_y_0, x = var_1310_cast_fp16, y = v_new_47_cast_fp16)[name = string("op_1311_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1312_cast_fp16 = add(x = var_1308_cast_fp16, y = var_1311_cast_fp16)[name = string("op_1312_cast_fp16")];
+            tensor<int32, [4]> var_1314_begin_0 = const()[name = string("op_1314_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_1314_end_0 = const()[name = string("op_1314_end_0"), val = tensor<int32, [4]>([32, 8, 1, 1])];
+            tensor<bool, [4]> var_1314_end_mask_0 = const()[name = string("op_1314_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1314_squeeze_mask_0 = const()[name = string("op_1314_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1314_cast_fp16 = slice_by_index(begin = var_1314_begin_0, end = var_1314_end_0, end_mask = var_1314_end_mask_0, squeeze_mask = var_1314_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1314_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1315_cast_fp16 = mul(x = state_47_cast_fp16, y = var_1314_cast_fp16)[name = string("op_1315_cast_fp16")];
+            tensor<int32, [4]> var_1317_begin_0 = const()[name = string("op_1317_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_1317_end_0 = const()[name = string("op_1317_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_1317_end_mask_0 = const()[name = string("op_1317_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1317_squeeze_mask_0 = const()[name = string("op_1317_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1317_cast_fp16 = slice_by_index(begin = var_1317_begin_0, end = var_1317_end_0, end_mask = var_1317_end_mask_0, squeeze_mask = var_1317_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1317_cast_fp16")];
+            bool var_1319_transpose_x_1 = const()[name = string("op_1319_transpose_x_1"), val = bool(true)];
+            bool var_1319_transpose_y_1 = const()[name = string("op_1319_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1319_cast_fp16 = matmul(transpose_x = var_1319_transpose_x_1, transpose_y = var_1319_transpose_y_1, x = var_1317_cast_fp16, y = v_new_47_cast_fp16)[name = string("op_1319_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_49_cast_fp16 = add(x = var_1315_cast_fp16, y = var_1319_cast_fp16)[name = string("state_49_cast_fp16")];
+            tensor<int32, [4]> var_1322_begin_0 = const()[name = string("op_1322_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_1322_end_0 = const()[name = string("op_1322_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_1322_end_mask_0 = const()[name = string("op_1322_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1322_squeeze_mask_0 = const()[name = string("op_1322_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1322_cast_fp16 = slice_by_index(begin = var_1322_begin_0, end = var_1322_end_0, end_mask = var_1322_end_mask_0, squeeze_mask = var_1322_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1322_cast_fp16")];
+            tensor<int32, [4]> var_1324_begin_0 = const()[name = string("op_1324_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_1324_end_0 = const()[name = string("op_1324_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_1324_end_mask_0 = const()[name = string("op_1324_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1324_squeeze_mask_0 = const()[name = string("op_1324_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1324_cast_fp16 = slice_by_index(begin = var_1324_begin_0, end = var_1324_end_0, end_mask = var_1324_end_mask_0, squeeze_mask = var_1324_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1324_cast_fp16")];
+            bool var_1325_transpose_x_0 = const()[name = string("op_1325_transpose_x_0"), val = bool(false)];
+            bool var_1325_transpose_y_0 = const()[name = string("op_1325_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1325_cast_fp16 = matmul(transpose_x = var_1325_transpose_x_0, transpose_y = var_1325_transpose_y_0, x = var_1324_cast_fp16, y = state_49_cast_fp16)[name = string("op_1325_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_49_cast_fp16 = sub(x = var_1322_cast_fp16, y = var_1325_cast_fp16)[name = string("v_new_49_cast_fp16")];
+            tensor<int32, [4]> var_1328_begin_0 = const()[name = string("op_1328_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_1328_end_0 = const()[name = string("op_1328_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_1328_end_mask_0 = const()[name = string("op_1328_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1328_squeeze_mask_0 = const()[name = string("op_1328_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1328_cast_fp16 = slice_by_index(begin = var_1328_begin_0, end = var_1328_end_0, end_mask = var_1328_end_mask_0, squeeze_mask = var_1328_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1328_cast_fp16")];
+            bool var_1329_transpose_x_0 = const()[name = string("op_1329_transpose_x_0"), val = bool(false)];
+            bool var_1329_transpose_y_0 = const()[name = string("op_1329_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1329_cast_fp16 = matmul(transpose_x = var_1329_transpose_x_0, transpose_y = var_1329_transpose_y_0, x = var_1328_cast_fp16, y = state_49_cast_fp16)[name = string("op_1329_cast_fp16")];
+            tensor<int32, [4]> var_1331_begin_0 = const()[name = string("op_1331_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_1331_end_0 = const()[name = string("op_1331_end_0"), val = tensor<int32, [4]>([32, 9, 64, 64])];
+            tensor<bool, [4]> var_1331_end_mask_0 = const()[name = string("op_1331_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1331_squeeze_mask_0 = const()[name = string("op_1331_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1331_cast_fp16 = slice_by_index(begin = var_1331_begin_0, end = var_1331_end_0, end_mask = var_1331_end_mask_0, squeeze_mask = var_1331_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1331_cast_fp16")];
+            bool var_1332_transpose_x_0 = const()[name = string("op_1332_transpose_x_0"), val = bool(false)];
+            bool var_1332_transpose_y_0 = const()[name = string("op_1332_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1332_cast_fp16 = matmul(transpose_x = var_1332_transpose_x_0, transpose_y = var_1332_transpose_y_0, x = var_1331_cast_fp16, y = v_new_49_cast_fp16)[name = string("op_1332_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1333_cast_fp16 = add(x = var_1329_cast_fp16, y = var_1332_cast_fp16)[name = string("op_1333_cast_fp16")];
+            tensor<int32, [4]> var_1335_begin_0 = const()[name = string("op_1335_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_1335_end_0 = const()[name = string("op_1335_end_0"), val = tensor<int32, [4]>([32, 9, 1, 1])];
+            tensor<bool, [4]> var_1335_end_mask_0 = const()[name = string("op_1335_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1335_squeeze_mask_0 = const()[name = string("op_1335_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1335_cast_fp16 = slice_by_index(begin = var_1335_begin_0, end = var_1335_end_0, end_mask = var_1335_end_mask_0, squeeze_mask = var_1335_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1335_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1336_cast_fp16 = mul(x = state_49_cast_fp16, y = var_1335_cast_fp16)[name = string("op_1336_cast_fp16")];
+            tensor<int32, [4]> var_1338_begin_0 = const()[name = string("op_1338_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_1338_end_0 = const()[name = string("op_1338_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_1338_end_mask_0 = const()[name = string("op_1338_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1338_squeeze_mask_0 = const()[name = string("op_1338_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1338_cast_fp16 = slice_by_index(begin = var_1338_begin_0, end = var_1338_end_0, end_mask = var_1338_end_mask_0, squeeze_mask = var_1338_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1338_cast_fp16")];
+            bool var_1340_transpose_x_1 = const()[name = string("op_1340_transpose_x_1"), val = bool(true)];
+            bool var_1340_transpose_y_1 = const()[name = string("op_1340_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1340_cast_fp16 = matmul(transpose_x = var_1340_transpose_x_1, transpose_y = var_1340_transpose_y_1, x = var_1338_cast_fp16, y = v_new_49_cast_fp16)[name = string("op_1340_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_51_cast_fp16 = add(x = var_1336_cast_fp16, y = var_1340_cast_fp16)[name = string("state_51_cast_fp16")];
+            tensor<int32, [4]> var_1343_begin_0 = const()[name = string("op_1343_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_1343_end_0 = const()[name = string("op_1343_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_1343_end_mask_0 = const()[name = string("op_1343_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1343_squeeze_mask_0 = const()[name = string("op_1343_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1343_cast_fp16 = slice_by_index(begin = var_1343_begin_0, end = var_1343_end_0, end_mask = var_1343_end_mask_0, squeeze_mask = var_1343_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1343_cast_fp16")];
+            tensor<int32, [4]> var_1345_begin_0 = const()[name = string("op_1345_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_1345_end_0 = const()[name = string("op_1345_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_1345_end_mask_0 = const()[name = string("op_1345_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1345_squeeze_mask_0 = const()[name = string("op_1345_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1345_cast_fp16 = slice_by_index(begin = var_1345_begin_0, end = var_1345_end_0, end_mask = var_1345_end_mask_0, squeeze_mask = var_1345_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1345_cast_fp16")];
+            bool var_1346_transpose_x_0 = const()[name = string("op_1346_transpose_x_0"), val = bool(false)];
+            bool var_1346_transpose_y_0 = const()[name = string("op_1346_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1346_cast_fp16 = matmul(transpose_x = var_1346_transpose_x_0, transpose_y = var_1346_transpose_y_0, x = var_1345_cast_fp16, y = state_51_cast_fp16)[name = string("op_1346_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_51_cast_fp16 = sub(x = var_1343_cast_fp16, y = var_1346_cast_fp16)[name = string("v_new_51_cast_fp16")];
+            tensor<int32, [4]> var_1349_begin_0 = const()[name = string("op_1349_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_1349_end_0 = const()[name = string("op_1349_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_1349_end_mask_0 = const()[name = string("op_1349_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1349_squeeze_mask_0 = const()[name = string("op_1349_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1349_cast_fp16 = slice_by_index(begin = var_1349_begin_0, end = var_1349_end_0, end_mask = var_1349_end_mask_0, squeeze_mask = var_1349_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1349_cast_fp16")];
+            bool var_1350_transpose_x_0 = const()[name = string("op_1350_transpose_x_0"), val = bool(false)];
+            bool var_1350_transpose_y_0 = const()[name = string("op_1350_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1350_cast_fp16 = matmul(transpose_x = var_1350_transpose_x_0, transpose_y = var_1350_transpose_y_0, x = var_1349_cast_fp16, y = state_51_cast_fp16)[name = string("op_1350_cast_fp16")];
+            tensor<int32, [4]> var_1352_begin_0 = const()[name = string("op_1352_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_1352_end_0 = const()[name = string("op_1352_end_0"), val = tensor<int32, [4]>([32, 10, 64, 64])];
+            tensor<bool, [4]> var_1352_end_mask_0 = const()[name = string("op_1352_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1352_squeeze_mask_0 = const()[name = string("op_1352_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1352_cast_fp16 = slice_by_index(begin = var_1352_begin_0, end = var_1352_end_0, end_mask = var_1352_end_mask_0, squeeze_mask = var_1352_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1352_cast_fp16")];
+            bool var_1353_transpose_x_0 = const()[name = string("op_1353_transpose_x_0"), val = bool(false)];
+            bool var_1353_transpose_y_0 = const()[name = string("op_1353_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1353_cast_fp16 = matmul(transpose_x = var_1353_transpose_x_0, transpose_y = var_1353_transpose_y_0, x = var_1352_cast_fp16, y = v_new_51_cast_fp16)[name = string("op_1353_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1354_cast_fp16 = add(x = var_1350_cast_fp16, y = var_1353_cast_fp16)[name = string("op_1354_cast_fp16")];
+            tensor<int32, [4]> var_1356_begin_0 = const()[name = string("op_1356_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_1356_end_0 = const()[name = string("op_1356_end_0"), val = tensor<int32, [4]>([32, 10, 1, 1])];
+            tensor<bool, [4]> var_1356_end_mask_0 = const()[name = string("op_1356_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1356_squeeze_mask_0 = const()[name = string("op_1356_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1356_cast_fp16 = slice_by_index(begin = var_1356_begin_0, end = var_1356_end_0, end_mask = var_1356_end_mask_0, squeeze_mask = var_1356_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1356_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1357_cast_fp16 = mul(x = state_51_cast_fp16, y = var_1356_cast_fp16)[name = string("op_1357_cast_fp16")];
+            tensor<int32, [4]> var_1359_begin_0 = const()[name = string("op_1359_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_1359_end_0 = const()[name = string("op_1359_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_1359_end_mask_0 = const()[name = string("op_1359_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1359_squeeze_mask_0 = const()[name = string("op_1359_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1359_cast_fp16 = slice_by_index(begin = var_1359_begin_0, end = var_1359_end_0, end_mask = var_1359_end_mask_0, squeeze_mask = var_1359_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1359_cast_fp16")];
+            bool var_1361_transpose_x_1 = const()[name = string("op_1361_transpose_x_1"), val = bool(true)];
+            bool var_1361_transpose_y_1 = const()[name = string("op_1361_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1361_cast_fp16 = matmul(transpose_x = var_1361_transpose_x_1, transpose_y = var_1361_transpose_y_1, x = var_1359_cast_fp16, y = v_new_51_cast_fp16)[name = string("op_1361_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_53_cast_fp16 = add(x = var_1357_cast_fp16, y = var_1361_cast_fp16)[name = string("state_53_cast_fp16")];
+            tensor<int32, [4]> var_1364_begin_0 = const()[name = string("op_1364_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_1364_end_0 = const()[name = string("op_1364_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_1364_end_mask_0 = const()[name = string("op_1364_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1364_squeeze_mask_0 = const()[name = string("op_1364_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1364_cast_fp16 = slice_by_index(begin = var_1364_begin_0, end = var_1364_end_0, end_mask = var_1364_end_mask_0, squeeze_mask = var_1364_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1364_cast_fp16")];
+            tensor<int32, [4]> var_1366_begin_0 = const()[name = string("op_1366_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_1366_end_0 = const()[name = string("op_1366_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_1366_end_mask_0 = const()[name = string("op_1366_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1366_squeeze_mask_0 = const()[name = string("op_1366_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1366_cast_fp16 = slice_by_index(begin = var_1366_begin_0, end = var_1366_end_0, end_mask = var_1366_end_mask_0, squeeze_mask = var_1366_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1366_cast_fp16")];
+            bool var_1367_transpose_x_0 = const()[name = string("op_1367_transpose_x_0"), val = bool(false)];
+            bool var_1367_transpose_y_0 = const()[name = string("op_1367_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1367_cast_fp16 = matmul(transpose_x = var_1367_transpose_x_0, transpose_y = var_1367_transpose_y_0, x = var_1366_cast_fp16, y = state_53_cast_fp16)[name = string("op_1367_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_53_cast_fp16 = sub(x = var_1364_cast_fp16, y = var_1367_cast_fp16)[name = string("v_new_53_cast_fp16")];
+            tensor<int32, [4]> var_1370_begin_0 = const()[name = string("op_1370_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_1370_end_0 = const()[name = string("op_1370_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_1370_end_mask_0 = const()[name = string("op_1370_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1370_squeeze_mask_0 = const()[name = string("op_1370_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1370_cast_fp16 = slice_by_index(begin = var_1370_begin_0, end = var_1370_end_0, end_mask = var_1370_end_mask_0, squeeze_mask = var_1370_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1370_cast_fp16")];
+            bool var_1371_transpose_x_0 = const()[name = string("op_1371_transpose_x_0"), val = bool(false)];
+            bool var_1371_transpose_y_0 = const()[name = string("op_1371_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1371_cast_fp16 = matmul(transpose_x = var_1371_transpose_x_0, transpose_y = var_1371_transpose_y_0, x = var_1370_cast_fp16, y = state_53_cast_fp16)[name = string("op_1371_cast_fp16")];
+            tensor<int32, [4]> var_1373_begin_0 = const()[name = string("op_1373_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_1373_end_0 = const()[name = string("op_1373_end_0"), val = tensor<int32, [4]>([32, 11, 64, 64])];
+            tensor<bool, [4]> var_1373_end_mask_0 = const()[name = string("op_1373_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1373_squeeze_mask_0 = const()[name = string("op_1373_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1373_cast_fp16 = slice_by_index(begin = var_1373_begin_0, end = var_1373_end_0, end_mask = var_1373_end_mask_0, squeeze_mask = var_1373_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1373_cast_fp16")];
+            bool var_1374_transpose_x_0 = const()[name = string("op_1374_transpose_x_0"), val = bool(false)];
+            bool var_1374_transpose_y_0 = const()[name = string("op_1374_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1374_cast_fp16 = matmul(transpose_x = var_1374_transpose_x_0, transpose_y = var_1374_transpose_y_0, x = var_1373_cast_fp16, y = v_new_53_cast_fp16)[name = string("op_1374_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1375_cast_fp16 = add(x = var_1371_cast_fp16, y = var_1374_cast_fp16)[name = string("op_1375_cast_fp16")];
+            tensor<int32, [4]> var_1377_begin_0 = const()[name = string("op_1377_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_1377_end_0 = const()[name = string("op_1377_end_0"), val = tensor<int32, [4]>([32, 11, 1, 1])];
+            tensor<bool, [4]> var_1377_end_mask_0 = const()[name = string("op_1377_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1377_squeeze_mask_0 = const()[name = string("op_1377_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1377_cast_fp16 = slice_by_index(begin = var_1377_begin_0, end = var_1377_end_0, end_mask = var_1377_end_mask_0, squeeze_mask = var_1377_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1377_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1378_cast_fp16 = mul(x = state_53_cast_fp16, y = var_1377_cast_fp16)[name = string("op_1378_cast_fp16")];
+            tensor<int32, [4]> var_1380_begin_0 = const()[name = string("op_1380_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_1380_end_0 = const()[name = string("op_1380_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_1380_end_mask_0 = const()[name = string("op_1380_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1380_squeeze_mask_0 = const()[name = string("op_1380_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1380_cast_fp16 = slice_by_index(begin = var_1380_begin_0, end = var_1380_end_0, end_mask = var_1380_end_mask_0, squeeze_mask = var_1380_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1380_cast_fp16")];
+            bool var_1382_transpose_x_1 = const()[name = string("op_1382_transpose_x_1"), val = bool(true)];
+            bool var_1382_transpose_y_1 = const()[name = string("op_1382_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1382_cast_fp16 = matmul(transpose_x = var_1382_transpose_x_1, transpose_y = var_1382_transpose_y_1, x = var_1380_cast_fp16, y = v_new_53_cast_fp16)[name = string("op_1382_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_55_cast_fp16 = add(x = var_1378_cast_fp16, y = var_1382_cast_fp16)[name = string("state_55_cast_fp16")];
+            tensor<int32, [4]> var_1385_begin_0 = const()[name = string("op_1385_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_1385_end_0 = const()[name = string("op_1385_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_1385_end_mask_0 = const()[name = string("op_1385_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1385_squeeze_mask_0 = const()[name = string("op_1385_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1385_cast_fp16 = slice_by_index(begin = var_1385_begin_0, end = var_1385_end_0, end_mask = var_1385_end_mask_0, squeeze_mask = var_1385_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1385_cast_fp16")];
+            tensor<int32, [4]> var_1387_begin_0 = const()[name = string("op_1387_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_1387_end_0 = const()[name = string("op_1387_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_1387_end_mask_0 = const()[name = string("op_1387_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1387_squeeze_mask_0 = const()[name = string("op_1387_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1387_cast_fp16 = slice_by_index(begin = var_1387_begin_0, end = var_1387_end_0, end_mask = var_1387_end_mask_0, squeeze_mask = var_1387_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1387_cast_fp16")];
+            bool var_1388_transpose_x_0 = const()[name = string("op_1388_transpose_x_0"), val = bool(false)];
+            bool var_1388_transpose_y_0 = const()[name = string("op_1388_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1388_cast_fp16 = matmul(transpose_x = var_1388_transpose_x_0, transpose_y = var_1388_transpose_y_0, x = var_1387_cast_fp16, y = state_55_cast_fp16)[name = string("op_1388_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_55_cast_fp16 = sub(x = var_1385_cast_fp16, y = var_1388_cast_fp16)[name = string("v_new_55_cast_fp16")];
+            tensor<int32, [4]> var_1391_begin_0 = const()[name = string("op_1391_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_1391_end_0 = const()[name = string("op_1391_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_1391_end_mask_0 = const()[name = string("op_1391_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1391_squeeze_mask_0 = const()[name = string("op_1391_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1391_cast_fp16 = slice_by_index(begin = var_1391_begin_0, end = var_1391_end_0, end_mask = var_1391_end_mask_0, squeeze_mask = var_1391_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1391_cast_fp16")];
+            bool var_1392_transpose_x_0 = const()[name = string("op_1392_transpose_x_0"), val = bool(false)];
+            bool var_1392_transpose_y_0 = const()[name = string("op_1392_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1392_cast_fp16 = matmul(transpose_x = var_1392_transpose_x_0, transpose_y = var_1392_transpose_y_0, x = var_1391_cast_fp16, y = state_55_cast_fp16)[name = string("op_1392_cast_fp16")];
+            tensor<int32, [4]> var_1394_begin_0 = const()[name = string("op_1394_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_1394_end_0 = const()[name = string("op_1394_end_0"), val = tensor<int32, [4]>([32, 12, 64, 64])];
+            tensor<bool, [4]> var_1394_end_mask_0 = const()[name = string("op_1394_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1394_squeeze_mask_0 = const()[name = string("op_1394_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1394_cast_fp16 = slice_by_index(begin = var_1394_begin_0, end = var_1394_end_0, end_mask = var_1394_end_mask_0, squeeze_mask = var_1394_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1394_cast_fp16")];
+            bool var_1395_transpose_x_0 = const()[name = string("op_1395_transpose_x_0"), val = bool(false)];
+            bool var_1395_transpose_y_0 = const()[name = string("op_1395_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1395_cast_fp16 = matmul(transpose_x = var_1395_transpose_x_0, transpose_y = var_1395_transpose_y_0, x = var_1394_cast_fp16, y = v_new_55_cast_fp16)[name = string("op_1395_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1396_cast_fp16 = add(x = var_1392_cast_fp16, y = var_1395_cast_fp16)[name = string("op_1396_cast_fp16")];
+            tensor<int32, [4]> var_1398_begin_0 = const()[name = string("op_1398_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_1398_end_0 = const()[name = string("op_1398_end_0"), val = tensor<int32, [4]>([32, 12, 1, 1])];
+            tensor<bool, [4]> var_1398_end_mask_0 = const()[name = string("op_1398_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1398_squeeze_mask_0 = const()[name = string("op_1398_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1398_cast_fp16 = slice_by_index(begin = var_1398_begin_0, end = var_1398_end_0, end_mask = var_1398_end_mask_0, squeeze_mask = var_1398_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1398_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1399_cast_fp16 = mul(x = state_55_cast_fp16, y = var_1398_cast_fp16)[name = string("op_1399_cast_fp16")];
+            tensor<int32, [4]> var_1401_begin_0 = const()[name = string("op_1401_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_1401_end_0 = const()[name = string("op_1401_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_1401_end_mask_0 = const()[name = string("op_1401_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1401_squeeze_mask_0 = const()[name = string("op_1401_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1401_cast_fp16 = slice_by_index(begin = var_1401_begin_0, end = var_1401_end_0, end_mask = var_1401_end_mask_0, squeeze_mask = var_1401_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1401_cast_fp16")];
+            bool var_1403_transpose_x_1 = const()[name = string("op_1403_transpose_x_1"), val = bool(true)];
+            bool var_1403_transpose_y_1 = const()[name = string("op_1403_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1403_cast_fp16 = matmul(transpose_x = var_1403_transpose_x_1, transpose_y = var_1403_transpose_y_1, x = var_1401_cast_fp16, y = v_new_55_cast_fp16)[name = string("op_1403_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_57_cast_fp16 = add(x = var_1399_cast_fp16, y = var_1403_cast_fp16)[name = string("state_57_cast_fp16")];
+            tensor<int32, [4]> var_1406_begin_0 = const()[name = string("op_1406_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_1406_end_0 = const()[name = string("op_1406_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_1406_end_mask_0 = const()[name = string("op_1406_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1406_squeeze_mask_0 = const()[name = string("op_1406_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1406_cast_fp16 = slice_by_index(begin = var_1406_begin_0, end = var_1406_end_0, end_mask = var_1406_end_mask_0, squeeze_mask = var_1406_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1406_cast_fp16")];
+            tensor<int32, [4]> var_1408_begin_0 = const()[name = string("op_1408_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_1408_end_0 = const()[name = string("op_1408_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_1408_end_mask_0 = const()[name = string("op_1408_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1408_squeeze_mask_0 = const()[name = string("op_1408_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1408_cast_fp16 = slice_by_index(begin = var_1408_begin_0, end = var_1408_end_0, end_mask = var_1408_end_mask_0, squeeze_mask = var_1408_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1408_cast_fp16")];
+            bool var_1409_transpose_x_0 = const()[name = string("op_1409_transpose_x_0"), val = bool(false)];
+            bool var_1409_transpose_y_0 = const()[name = string("op_1409_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1409_cast_fp16 = matmul(transpose_x = var_1409_transpose_x_0, transpose_y = var_1409_transpose_y_0, x = var_1408_cast_fp16, y = state_57_cast_fp16)[name = string("op_1409_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_57_cast_fp16 = sub(x = var_1406_cast_fp16, y = var_1409_cast_fp16)[name = string("v_new_57_cast_fp16")];
+            tensor<int32, [4]> var_1412_begin_0 = const()[name = string("op_1412_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_1412_end_0 = const()[name = string("op_1412_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_1412_end_mask_0 = const()[name = string("op_1412_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1412_squeeze_mask_0 = const()[name = string("op_1412_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1412_cast_fp16 = slice_by_index(begin = var_1412_begin_0, end = var_1412_end_0, end_mask = var_1412_end_mask_0, squeeze_mask = var_1412_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1412_cast_fp16")];
+            bool var_1413_transpose_x_0 = const()[name = string("op_1413_transpose_x_0"), val = bool(false)];
+            bool var_1413_transpose_y_0 = const()[name = string("op_1413_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1413_cast_fp16 = matmul(transpose_x = var_1413_transpose_x_0, transpose_y = var_1413_transpose_y_0, x = var_1412_cast_fp16, y = state_57_cast_fp16)[name = string("op_1413_cast_fp16")];
+            tensor<int32, [4]> var_1415_begin_0 = const()[name = string("op_1415_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_1415_end_0 = const()[name = string("op_1415_end_0"), val = tensor<int32, [4]>([32, 13, 64, 64])];
+            tensor<bool, [4]> var_1415_end_mask_0 = const()[name = string("op_1415_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1415_squeeze_mask_0 = const()[name = string("op_1415_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1415_cast_fp16 = slice_by_index(begin = var_1415_begin_0, end = var_1415_end_0, end_mask = var_1415_end_mask_0, squeeze_mask = var_1415_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1415_cast_fp16")];
+            bool var_1416_transpose_x_0 = const()[name = string("op_1416_transpose_x_0"), val = bool(false)];
+            bool var_1416_transpose_y_0 = const()[name = string("op_1416_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1416_cast_fp16 = matmul(transpose_x = var_1416_transpose_x_0, transpose_y = var_1416_transpose_y_0, x = var_1415_cast_fp16, y = v_new_57_cast_fp16)[name = string("op_1416_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1417_cast_fp16 = add(x = var_1413_cast_fp16, y = var_1416_cast_fp16)[name = string("op_1417_cast_fp16")];
+            tensor<int32, [4]> var_1419_begin_0 = const()[name = string("op_1419_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_1419_end_0 = const()[name = string("op_1419_end_0"), val = tensor<int32, [4]>([32, 13, 1, 1])];
+            tensor<bool, [4]> var_1419_end_mask_0 = const()[name = string("op_1419_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1419_squeeze_mask_0 = const()[name = string("op_1419_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1419_cast_fp16 = slice_by_index(begin = var_1419_begin_0, end = var_1419_end_0, end_mask = var_1419_end_mask_0, squeeze_mask = var_1419_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1419_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1420_cast_fp16 = mul(x = state_57_cast_fp16, y = var_1419_cast_fp16)[name = string("op_1420_cast_fp16")];
+            tensor<int32, [4]> var_1422_begin_0 = const()[name = string("op_1422_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_1422_end_0 = const()[name = string("op_1422_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_1422_end_mask_0 = const()[name = string("op_1422_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1422_squeeze_mask_0 = const()[name = string("op_1422_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1422_cast_fp16 = slice_by_index(begin = var_1422_begin_0, end = var_1422_end_0, end_mask = var_1422_end_mask_0, squeeze_mask = var_1422_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1422_cast_fp16")];
+            bool var_1424_transpose_x_1 = const()[name = string("op_1424_transpose_x_1"), val = bool(true)];
+            bool var_1424_transpose_y_1 = const()[name = string("op_1424_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1424_cast_fp16 = matmul(transpose_x = var_1424_transpose_x_1, transpose_y = var_1424_transpose_y_1, x = var_1422_cast_fp16, y = v_new_57_cast_fp16)[name = string("op_1424_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_59_cast_fp16 = add(x = var_1420_cast_fp16, y = var_1424_cast_fp16)[name = string("state_59_cast_fp16")];
+            tensor<int32, [4]> var_1427_begin_0 = const()[name = string("op_1427_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_1427_end_0 = const()[name = string("op_1427_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_1427_end_mask_0 = const()[name = string("op_1427_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1427_squeeze_mask_0 = const()[name = string("op_1427_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1427_cast_fp16 = slice_by_index(begin = var_1427_begin_0, end = var_1427_end_0, end_mask = var_1427_end_mask_0, squeeze_mask = var_1427_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1427_cast_fp16")];
+            tensor<int32, [4]> var_1429_begin_0 = const()[name = string("op_1429_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_1429_end_0 = const()[name = string("op_1429_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_1429_end_mask_0 = const()[name = string("op_1429_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1429_squeeze_mask_0 = const()[name = string("op_1429_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1429_cast_fp16 = slice_by_index(begin = var_1429_begin_0, end = var_1429_end_0, end_mask = var_1429_end_mask_0, squeeze_mask = var_1429_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1429_cast_fp16")];
+            bool var_1430_transpose_x_0 = const()[name = string("op_1430_transpose_x_0"), val = bool(false)];
+            bool var_1430_transpose_y_0 = const()[name = string("op_1430_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1430_cast_fp16 = matmul(transpose_x = var_1430_transpose_x_0, transpose_y = var_1430_transpose_y_0, x = var_1429_cast_fp16, y = state_59_cast_fp16)[name = string("op_1430_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_59_cast_fp16 = sub(x = var_1427_cast_fp16, y = var_1430_cast_fp16)[name = string("v_new_59_cast_fp16")];
+            tensor<int32, [4]> var_1433_begin_0 = const()[name = string("op_1433_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_1433_end_0 = const()[name = string("op_1433_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_1433_end_mask_0 = const()[name = string("op_1433_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1433_squeeze_mask_0 = const()[name = string("op_1433_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1433_cast_fp16 = slice_by_index(begin = var_1433_begin_0, end = var_1433_end_0, end_mask = var_1433_end_mask_0, squeeze_mask = var_1433_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1433_cast_fp16")];
+            bool var_1434_transpose_x_0 = const()[name = string("op_1434_transpose_x_0"), val = bool(false)];
+            bool var_1434_transpose_y_0 = const()[name = string("op_1434_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1434_cast_fp16 = matmul(transpose_x = var_1434_transpose_x_0, transpose_y = var_1434_transpose_y_0, x = var_1433_cast_fp16, y = state_59_cast_fp16)[name = string("op_1434_cast_fp16")];
+            tensor<int32, [4]> var_1436_begin_0 = const()[name = string("op_1436_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_1436_end_0 = const()[name = string("op_1436_end_0"), val = tensor<int32, [4]>([32, 14, 64, 64])];
+            tensor<bool, [4]> var_1436_end_mask_0 = const()[name = string("op_1436_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1436_squeeze_mask_0 = const()[name = string("op_1436_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1436_cast_fp16 = slice_by_index(begin = var_1436_begin_0, end = var_1436_end_0, end_mask = var_1436_end_mask_0, squeeze_mask = var_1436_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1436_cast_fp16")];
+            bool var_1437_transpose_x_0 = const()[name = string("op_1437_transpose_x_0"), val = bool(false)];
+            bool var_1437_transpose_y_0 = const()[name = string("op_1437_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1437_cast_fp16 = matmul(transpose_x = var_1437_transpose_x_0, transpose_y = var_1437_transpose_y_0, x = var_1436_cast_fp16, y = v_new_59_cast_fp16)[name = string("op_1437_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1438_cast_fp16 = add(x = var_1434_cast_fp16, y = var_1437_cast_fp16)[name = string("op_1438_cast_fp16")];
+            tensor<int32, [4]> var_1440_begin_0 = const()[name = string("op_1440_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_1440_end_0 = const()[name = string("op_1440_end_0"), val = tensor<int32, [4]>([32, 14, 1, 1])];
+            tensor<bool, [4]> var_1440_end_mask_0 = const()[name = string("op_1440_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1440_squeeze_mask_0 = const()[name = string("op_1440_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1440_cast_fp16 = slice_by_index(begin = var_1440_begin_0, end = var_1440_end_0, end_mask = var_1440_end_mask_0, squeeze_mask = var_1440_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1440_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1441_cast_fp16 = mul(x = state_59_cast_fp16, y = var_1440_cast_fp16)[name = string("op_1441_cast_fp16")];
+            tensor<int32, [4]> var_1443_begin_0 = const()[name = string("op_1443_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_1443_end_0 = const()[name = string("op_1443_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_1443_end_mask_0 = const()[name = string("op_1443_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1443_squeeze_mask_0 = const()[name = string("op_1443_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1443_cast_fp16 = slice_by_index(begin = var_1443_begin_0, end = var_1443_end_0, end_mask = var_1443_end_mask_0, squeeze_mask = var_1443_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1443_cast_fp16")];
+            bool var_1445_transpose_x_1 = const()[name = string("op_1445_transpose_x_1"), val = bool(true)];
+            bool var_1445_transpose_y_1 = const()[name = string("op_1445_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1445_cast_fp16 = matmul(transpose_x = var_1445_transpose_x_1, transpose_y = var_1445_transpose_y_1, x = var_1443_cast_fp16, y = v_new_59_cast_fp16)[name = string("op_1445_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_61_cast_fp16 = add(x = var_1441_cast_fp16, y = var_1445_cast_fp16)[name = string("state_61_cast_fp16")];
+            tensor<int32, [4]> var_1448_begin_0 = const()[name = string("op_1448_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_1448_end_0 = const()[name = string("op_1448_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_1448_end_mask_0 = const()[name = string("op_1448_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1448_squeeze_mask_0 = const()[name = string("op_1448_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1448_cast_fp16 = slice_by_index(begin = var_1448_begin_0, end = var_1448_end_0, end_mask = var_1448_end_mask_0, squeeze_mask = var_1448_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1448_cast_fp16")];
+            tensor<int32, [4]> var_1450_begin_0 = const()[name = string("op_1450_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_1450_end_0 = const()[name = string("op_1450_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_1450_end_mask_0 = const()[name = string("op_1450_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1450_squeeze_mask_0 = const()[name = string("op_1450_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1450_cast_fp16 = slice_by_index(begin = var_1450_begin_0, end = var_1450_end_0, end_mask = var_1450_end_mask_0, squeeze_mask = var_1450_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1450_cast_fp16")];
+            bool var_1451_transpose_x_0 = const()[name = string("op_1451_transpose_x_0"), val = bool(false)];
+            bool var_1451_transpose_y_0 = const()[name = string("op_1451_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1451_cast_fp16 = matmul(transpose_x = var_1451_transpose_x_0, transpose_y = var_1451_transpose_y_0, x = var_1450_cast_fp16, y = state_61_cast_fp16)[name = string("op_1451_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_61_cast_fp16 = sub(x = var_1448_cast_fp16, y = var_1451_cast_fp16)[name = string("v_new_61_cast_fp16")];
+            tensor<int32, [4]> var_1454_begin_0 = const()[name = string("op_1454_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_1454_end_0 = const()[name = string("op_1454_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_1454_end_mask_0 = const()[name = string("op_1454_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1454_squeeze_mask_0 = const()[name = string("op_1454_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1454_cast_fp16 = slice_by_index(begin = var_1454_begin_0, end = var_1454_end_0, end_mask = var_1454_end_mask_0, squeeze_mask = var_1454_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1454_cast_fp16")];
+            bool var_1455_transpose_x_0 = const()[name = string("op_1455_transpose_x_0"), val = bool(false)];
+            bool var_1455_transpose_y_0 = const()[name = string("op_1455_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1455_cast_fp16 = matmul(transpose_x = var_1455_transpose_x_0, transpose_y = var_1455_transpose_y_0, x = var_1454_cast_fp16, y = state_61_cast_fp16)[name = string("op_1455_cast_fp16")];
+            tensor<int32, [4]> var_1457_begin_0 = const()[name = string("op_1457_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_1457_end_0 = const()[name = string("op_1457_end_0"), val = tensor<int32, [4]>([32, 15, 64, 64])];
+            tensor<bool, [4]> var_1457_end_mask_0 = const()[name = string("op_1457_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1457_squeeze_mask_0 = const()[name = string("op_1457_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1457_cast_fp16 = slice_by_index(begin = var_1457_begin_0, end = var_1457_end_0, end_mask = var_1457_end_mask_0, squeeze_mask = var_1457_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1457_cast_fp16")];
+            bool var_1458_transpose_x_0 = const()[name = string("op_1458_transpose_x_0"), val = bool(false)];
+            bool var_1458_transpose_y_0 = const()[name = string("op_1458_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1458_cast_fp16 = matmul(transpose_x = var_1458_transpose_x_0, transpose_y = var_1458_transpose_y_0, x = var_1457_cast_fp16, y = v_new_61_cast_fp16)[name = string("op_1458_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1459_cast_fp16 = add(x = var_1455_cast_fp16, y = var_1458_cast_fp16)[name = string("op_1459_cast_fp16")];
+            tensor<int32, [4]> var_1461_begin_0 = const()[name = string("op_1461_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_1461_end_0 = const()[name = string("op_1461_end_0"), val = tensor<int32, [4]>([32, 15, 1, 1])];
+            tensor<bool, [4]> var_1461_end_mask_0 = const()[name = string("op_1461_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1461_squeeze_mask_0 = const()[name = string("op_1461_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1461_cast_fp16 = slice_by_index(begin = var_1461_begin_0, end = var_1461_end_0, end_mask = var_1461_end_mask_0, squeeze_mask = var_1461_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1461_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1462_cast_fp16 = mul(x = state_61_cast_fp16, y = var_1461_cast_fp16)[name = string("op_1462_cast_fp16")];
+            tensor<int32, [4]> var_1464_begin_0 = const()[name = string("op_1464_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_1464_end_0 = const()[name = string("op_1464_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_1464_end_mask_0 = const()[name = string("op_1464_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1464_squeeze_mask_0 = const()[name = string("op_1464_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1464_cast_fp16 = slice_by_index(begin = var_1464_begin_0, end = var_1464_end_0, end_mask = var_1464_end_mask_0, squeeze_mask = var_1464_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1464_cast_fp16")];
+            bool var_1466_transpose_x_1 = const()[name = string("op_1466_transpose_x_1"), val = bool(true)];
+            bool var_1466_transpose_y_1 = const()[name = string("op_1466_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1466_cast_fp16 = matmul(transpose_x = var_1466_transpose_x_1, transpose_y = var_1466_transpose_y_1, x = var_1464_cast_fp16, y = v_new_61_cast_fp16)[name = string("op_1466_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_63_cast_fp16 = add(x = var_1462_cast_fp16, y = var_1466_cast_fp16)[name = string("state_63_cast_fp16")];
+            tensor<int32, [4]> var_1469_begin_0 = const()[name = string("op_1469_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_1469_end_0 = const()[name = string("op_1469_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_1469_end_mask_0 = const()[name = string("op_1469_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1469_squeeze_mask_0 = const()[name = string("op_1469_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1469_cast_fp16 = slice_by_index(begin = var_1469_begin_0, end = var_1469_end_0, end_mask = var_1469_end_mask_0, squeeze_mask = var_1469_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1469_cast_fp16")];
+            tensor<int32, [4]> var_1471_begin_0 = const()[name = string("op_1471_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_1471_end_0 = const()[name = string("op_1471_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_1471_end_mask_0 = const()[name = string("op_1471_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1471_squeeze_mask_0 = const()[name = string("op_1471_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1471_cast_fp16 = slice_by_index(begin = var_1471_begin_0, end = var_1471_end_0, end_mask = var_1471_end_mask_0, squeeze_mask = var_1471_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1471_cast_fp16")];
+            bool var_1472_transpose_x_0 = const()[name = string("op_1472_transpose_x_0"), val = bool(false)];
+            bool var_1472_transpose_y_0 = const()[name = string("op_1472_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1472_cast_fp16 = matmul(transpose_x = var_1472_transpose_x_0, transpose_y = var_1472_transpose_y_0, x = var_1471_cast_fp16, y = state_63_cast_fp16)[name = string("op_1472_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_63_cast_fp16 = sub(x = var_1469_cast_fp16, y = var_1472_cast_fp16)[name = string("v_new_63_cast_fp16")];
+            tensor<int32, [4]> var_1475_begin_0 = const()[name = string("op_1475_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_1475_end_0 = const()[name = string("op_1475_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_1475_end_mask_0 = const()[name = string("op_1475_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1475_squeeze_mask_0 = const()[name = string("op_1475_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1475_cast_fp16 = slice_by_index(begin = var_1475_begin_0, end = var_1475_end_0, end_mask = var_1475_end_mask_0, squeeze_mask = var_1475_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1475_cast_fp16")];
+            bool var_1476_transpose_x_0 = const()[name = string("op_1476_transpose_x_0"), val = bool(false)];
+            bool var_1476_transpose_y_0 = const()[name = string("op_1476_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1476_cast_fp16 = matmul(transpose_x = var_1476_transpose_x_0, transpose_y = var_1476_transpose_y_0, x = var_1475_cast_fp16, y = state_63_cast_fp16)[name = string("op_1476_cast_fp16")];
+            tensor<int32, [4]> var_1478_begin_0 = const()[name = string("op_1478_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_1478_end_0 = const()[name = string("op_1478_end_0"), val = tensor<int32, [4]>([32, 16, 64, 64])];
+            tensor<bool, [4]> var_1478_end_mask_0 = const()[name = string("op_1478_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1478_squeeze_mask_0 = const()[name = string("op_1478_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1478_cast_fp16 = slice_by_index(begin = var_1478_begin_0, end = var_1478_end_0, end_mask = var_1478_end_mask_0, squeeze_mask = var_1478_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1478_cast_fp16")];
+            bool var_1479_transpose_x_0 = const()[name = string("op_1479_transpose_x_0"), val = bool(false)];
+            bool var_1479_transpose_y_0 = const()[name = string("op_1479_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1479_cast_fp16 = matmul(transpose_x = var_1479_transpose_x_0, transpose_y = var_1479_transpose_y_0, x = var_1478_cast_fp16, y = v_new_63_cast_fp16)[name = string("op_1479_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1480_cast_fp16 = add(x = var_1476_cast_fp16, y = var_1479_cast_fp16)[name = string("op_1480_cast_fp16")];
+            int32 var_1482_axis_0 = const()[name = string("op_1482_axis_0"), val = int32(1)];
+            tensor<fp16, [32, 16, 64, 128]> var_1482_cast_fp16 = stack(axis = var_1482_axis_0, values = (var_1165_cast_fp16, var_1186_cast_fp16, var_1207_cast_fp16, var_1228_cast_fp16, var_1249_cast_fp16, var_1270_cast_fp16, var_1291_cast_fp16, var_1312_cast_fp16, var_1333_cast_fp16, var_1354_cast_fp16, var_1375_cast_fp16, var_1396_cast_fp16, var_1417_cast_fp16, var_1438_cast_fp16, var_1459_cast_fp16, var_1480_cast_fp16))[name = string("op_1482_cast_fp16")];
+            tensor<int32, [3]> var_1483 = const()[name = string("op_1483"), val = tensor<int32, [3]>([32, 1024, 128])];
+            tensor<fp16, [32, 1024, 128]> var_1484_cast_fp16 = reshape(shape = var_1483, x = var_1482_cast_fp16)[name = string("op_1484_cast_fp16")];
+            tensor<int32, [3]> var_1485 = const()[name = string("op_1485"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> input_23_begin_0 = const()[name = string("input_23_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> input_23_end_0 = const()[name = string("input_23_end_0"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<bool, [4]> input_23_end_mask_0 = const()[name = string("input_23_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> input_23_squeeze_mask_0 = const()[name = string("input_23_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [1024, 32, 128]> input_23_cast_fp16 = slice_by_index(begin = input_23_begin_0, end = input_23_end_0, end_mask = input_23_end_mask_0, squeeze_mask = input_23_squeeze_mask_0, x = z_3_cast_fp16)[name = string("input_23_cast_fp16")];
+            fp16 var_801_promoted_1_to_fp16 = const()[name = string("op_801_promoted_1_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1024, 32, 128]> x_43_cast_fp16 = transpose(perm = var_1485, x = var_1484_cast_fp16)[name = string("transpose_40")];
+            tensor<fp16, [1024, 32, 128]> var_1489_cast_fp16 = pow(x = x_43_cast_fp16, y = var_801_promoted_1_to_fp16)[name = string("op_1489_cast_fp16")];
+            tensor<int32, [1]> var_1491_axes_0 = const()[name = string("op_1491_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_1491_keep_dims_0 = const()[name = string("op_1491_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1024, 32, 1]> var_1491_cast_fp16 = reduce_mean(axes = var_1491_axes_0, keep_dims = var_1491_keep_dims_0, x = var_1489_cast_fp16)[name = string("op_1491_cast_fp16")];
+            fp16 var_1492_to_fp16 = const()[name = string("op_1492_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1024, 32, 1]> var_1493_cast_fp16 = add(x = var_1491_cast_fp16, y = var_1492_to_fp16)[name = string("op_1493_cast_fp16")];
+            fp32 var_1494_epsilon_0 = const()[name = string("op_1494_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1024, 32, 1]> var_1494_cast_fp16 = rsqrt(epsilon = var_1494_epsilon_0, x = var_1493_cast_fp16)[name = string("op_1494_cast_fp16")];
+            tensor<fp16, [1024, 32, 128]> x_45_cast_fp16 = mul(x = x_43_cast_fp16, y = var_1494_cast_fp16)[name = string("x_45_cast_fp16")];
+            tensor<fp16, [128]> layers_1_linear_attn_norm_weight_to_fp16 = const()[name = string("layers_1_linear_attn_norm_weight_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(685602688)))];
+            tensor<fp16, [1024, 32, 128]> var_1496_cast_fp16 = mul(x = layers_1_linear_attn_norm_weight_to_fp16, y = x_45_cast_fp16)[name = string("op_1496_cast_fp16")];
+            tensor<fp16, [1024, 32, 128]> var_1497_cast_fp16 = silu(x = input_23_cast_fp16)[name = string("op_1497_cast_fp16")];
+            tensor<fp16, [1024, 32, 128]> core_3_cast_fp16 = mul(x = var_1496_cast_fp16, y = var_1497_cast_fp16)[name = string("core_3_cast_fp16")];
+            tensor<int32, [3]> var_1499 = const()[name = string("op_1499"), val = tensor<int32, [3]>([1, 1024, 4096])];
+            tensor<fp16, [1, 1024, 4096]> input_25_cast_fp16 = reshape(shape = var_1499, x = core_3_cast_fp16)[name = string("input_25_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> linear_12_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers_1_linear_attn_out_proj_weight_cast_fp16, x = input_25_cast_fp16)[name = string("linear_12_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> x_47_cast_fp16 = add(x = x_25_cast_fp16, y = linear_12_cast_fp16)[name = string("x_47_cast_fp16")];
+            fp16 var_801_promoted_2_to_fp16 = const()[name = string("op_801_promoted_2_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 1024, 2560]> var_1505_cast_fp16 = pow(x = x_47_cast_fp16, y = var_801_promoted_2_to_fp16)[name = string("op_1505_cast_fp16")];
+            tensor<int32, [1]> var_1507_axes_0 = const()[name = string("op_1507_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_1507_keep_dims_0 = const()[name = string("op_1507_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 1024, 1]> var_1507_cast_fp16 = reduce_mean(axes = var_1507_axes_0, keep_dims = var_1507_keep_dims_0, x = var_1505_cast_fp16)[name = string("op_1507_cast_fp16")];
+            fp16 var_1508_to_fp16 = const()[name = string("op_1508_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 1024, 1]> var_1509_cast_fp16 = add(x = var_1507_cast_fp16, y = var_1508_to_fp16)[name = string("op_1509_cast_fp16")];
+            fp32 var_1510_epsilon_0 = const()[name = string("op_1510_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 1024, 1]> var_1510_cast_fp16 = rsqrt(epsilon = var_1510_epsilon_0, x = var_1509_cast_fp16)[name = string("op_1510_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> x_49_cast_fp16 = mul(x = x_47_cast_fp16, y = var_1510_cast_fp16)[name = string("x_49_cast_fp16")];
+            tensor<fp16, [2560]> var_1513_to_fp16 = const()[name = string("op_1513_to_fp16"), val = tensor<fp16, [2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(685603008)))];
+            tensor<fp16, [1, 1024, 2560]> input_27_cast_fp16 = mul(x = x_49_cast_fp16, y = var_1513_to_fp16)[name = string("input_27_cast_fp16")];
+            tensor<fp16, [1, 1024, 9216]> linear_13_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = layers_1_mlp_gate_proj_weight_cast_fp16, x = input_27_cast_fp16)[name = string("linear_13_cast_fp16")];
+            tensor<fp16, [1, 1024, 9216]> var_1520_cast_fp16 = silu(x = linear_13_cast_fp16)[name = string("op_1520_cast_fp16")];
+            tensor<fp16, [1, 1024, 9216]> linear_14_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = layers_1_mlp_up_proj_weight_cast_fp16, x = input_27_cast_fp16)[name = string("linear_14_cast_fp16")];
+            tensor<fp16, [1, 1024, 9216]> input_31_cast_fp16 = mul(x = var_1520_cast_fp16, y = linear_14_cast_fp16)[name = string("input_31_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> linear_15_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers_1_mlp_down_proj_weight_cast_fp16, x = input_31_cast_fp16)[name = string("linear_15_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> x_51_cast_fp16 = add(x = x_47_cast_fp16, y = linear_15_cast_fp16)[name = string("x_51_cast_fp16")];
+            int32 var_1540 = const()[name = string("op_1540"), val = int32(-2)];
+            int32 var_1555 = const()[name = string("op_1555"), val = int32(-1)];
+            fp16 var_1554_promoted_to_fp16 = const()[name = string("op_1554_promoted_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 1024, 2560]> var_1564_cast_fp16 = pow(x = x_51_cast_fp16, y = var_1554_promoted_to_fp16)[name = string("op_1564_cast_fp16")];
+            tensor<int32, [1]> var_1566_axes_0 = const()[name = string("op_1566_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_1566_keep_dims_0 = const()[name = string("op_1566_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 1024, 1]> var_1566_cast_fp16 = reduce_mean(axes = var_1566_axes_0, keep_dims = var_1566_keep_dims_0, x = var_1564_cast_fp16)[name = string("op_1566_cast_fp16")];
+            fp16 var_1567_to_fp16 = const()[name = string("op_1567_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 1024, 1]> var_1568_cast_fp16 = add(x = var_1566_cast_fp16, y = var_1567_to_fp16)[name = string("op_1568_cast_fp16")];
+            fp32 var_1569_epsilon_0 = const()[name = string("op_1569_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 1024, 1]> var_1569_cast_fp16 = rsqrt(epsilon = var_1569_epsilon_0, x = var_1568_cast_fp16)[name = string("op_1569_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> x_53_cast_fp16 = mul(x = x_51_cast_fp16, y = var_1569_cast_fp16)[name = string("x_53_cast_fp16")];
+            tensor<fp16, [2560]> var_1572_to_fp16 = const()[name = string("op_1572_to_fp16"), val = tensor<fp16, [2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(685608192)))];
+            tensor<fp16, [1, 1024, 2560]> x_55_cast_fp16 = mul(x = x_53_cast_fp16, y = var_1572_to_fp16)[name = string("x_55_cast_fp16")];
+            tensor<fp16, [1, 1024, 8192]> linear_16_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = layers_2_linear_attn_in_proj_qkv_weight_cast_fp16, x = x_55_cast_fp16)[name = string("linear_16_cast_fp16")];
+            tensor<int32, [3]> input_33_perm_0 = const()[name = string("input_33_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
+            tensor<int32, [6]> input_35_pad_0 = const()[name = string("input_35_pad_0"), val = tensor<int32, [6]>([0, 0, 0, 0, 3, 0])];
+            string input_35_mode_0 = const()[name = string("input_35_mode_0"), val = string("constant")];
+            fp16 const_25_to_fp16 = const()[name = string("const_25_to_fp16"), val = fp16(0x0p+0)];
+            tensor<fp16, [1, 8192, 1024]> input_33_cast_fp16 = transpose(perm = input_33_perm_0, x = linear_16_cast_fp16)[name = string("transpose_39")];
+            tensor<fp16, [1, 8192, 1027]> input_35_cast_fp16 = pad(constant_val = const_25_to_fp16, mode = input_35_mode_0, pad = input_35_pad_0, x = input_33_cast_fp16)[name = string("input_35_cast_fp16")];
+            string input_37_pad_type_0 = const()[name = string("input_37_pad_type_0"), val = string("valid")];
+            int32 input_37_groups_0 = const()[name = string("input_37_groups_0"), val = int32(8192)];
+            tensor<int32, [1]> input_37_strides_0 = const()[name = string("input_37_strides_0"), val = tensor<int32, [1]>([1])];
+            tensor<int32, [2]> input_37_pad_0 = const()[name = string("input_37_pad_0"), val = tensor<int32, [2]>([0, 0])];
+            tensor<int32, [1]> input_37_dilations_0 = const()[name = string("input_37_dilations_0"), val = tensor<int32, [1]>([1])];
+            tensor<fp16, [8192, 1, 4]> layers_2_linear_attn_conv1d_weight_to_fp16 = const()[name = string("layers_2_linear_attn_conv1d_weight_to_fp16"), val = tensor<fp16, [8192, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(685613376)))];
+            tensor<fp16, [1, 8192, 1024]> input_37_cast_fp16 = conv(dilations = input_37_dilations_0, groups = input_37_groups_0, pad = input_37_pad_0, pad_type = input_37_pad_type_0, strides = input_37_strides_0, weight = layers_2_linear_attn_conv1d_weight_to_fp16, x = input_35_cast_fp16)[name = string("input_37_cast_fp16")];
+            tensor<fp16, [1, 8192, 1024]> var_1607_cast_fp16 = silu(x = input_37_cast_fp16)[name = string("op_1607_cast_fp16")];
+            tensor<int32, [3]> qkv_5_perm_0 = const()[name = string("qkv_5_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
+            tensor<int32, [3]> var_1609_begin_0 = const()[name = string("op_1609_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_1609_end_0 = const()[name = string("op_1609_end_0"), val = tensor<int32, [3]>([1, 1024, 2048])];
+            tensor<bool, [3]> var_1609_end_mask_0 = const()[name = string("op_1609_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
+            tensor<fp16, [1, 1024, 8192]> qkv_5_cast_fp16 = transpose(perm = qkv_5_perm_0, x = var_1607_cast_fp16)[name = string("transpose_38")];
+            tensor<fp16, [1, 1024, 2048]> var_1609_cast_fp16 = slice_by_index(begin = var_1609_begin_0, end = var_1609_end_0, end_mask = var_1609_end_mask_0, x = qkv_5_cast_fp16)[name = string("op_1609_cast_fp16")];
+            tensor<int32, [4]> var_1610 = const()[name = string("op_1610"), val = tensor<int32, [4]>([1, 1024, 16, 128])];
+            tensor<fp16, [1, 1024, 16, 128]> q_21_cast_fp16 = reshape(shape = var_1610, x = var_1609_cast_fp16)[name = string("q_21_cast_fp16")];
+            tensor<int32, [3]> var_1612_begin_0 = const()[name = string("op_1612_begin_0"), val = tensor<int32, [3]>([0, 0, 2048])];
+            tensor<int32, [3]> var_1612_end_0 = const()[name = string("op_1612_end_0"), val = tensor<int32, [3]>([1, 1024, 4096])];
+            tensor<bool, [3]> var_1612_end_mask_0 = const()[name = string("op_1612_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
+            tensor<fp16, [1, 1024, 2048]> var_1612_cast_fp16 = slice_by_index(begin = var_1612_begin_0, end = var_1612_end_0, end_mask = var_1612_end_mask_0, x = qkv_5_cast_fp16)[name = string("op_1612_cast_fp16")];
+            tensor<int32, [4]> var_1613 = const()[name = string("op_1613"), val = tensor<int32, [4]>([1, 1024, 16, 128])];
+            tensor<fp16, [1, 1024, 16, 128]> k_17_cast_fp16 = reshape(shape = var_1613, x = var_1612_cast_fp16)[name = string("k_17_cast_fp16")];
+            tensor<int32, [3]> var_1615_begin_0 = const()[name = string("op_1615_begin_0"), val = tensor<int32, [3]>([0, 0, 4096])];
+            tensor<int32, [3]> var_1615_end_0 = const()[name = string("op_1615_end_0"), val = tensor<int32, [3]>([1, 1024, 8192])];
+            tensor<bool, [3]> var_1615_end_mask_0 = const()[name = string("op_1615_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
+            tensor<fp16, [1, 1024, 4096]> var_1615_cast_fp16 = slice_by_index(begin = var_1615_begin_0, end = var_1615_end_0, end_mask = var_1615_end_mask_0, x = qkv_5_cast_fp16)[name = string("op_1615_cast_fp16")];
+            tensor<int32, [4]> var_1616 = const()[name = string("op_1616"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<fp16, [1, 1024, 32, 128]> v_9_cast_fp16 = reshape(shape = var_1616, x = var_1615_cast_fp16)[name = string("v_9_cast_fp16")];
+            tensor<fp16, [1, 1024, 4096]> linear_17_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = layers_2_linear_attn_in_proj_z_weight_cast_fp16, x = x_55_cast_fp16)[name = string("linear_17_cast_fp16")];
+            tensor<int32, [4]> var_1620 = const()[name = string("op_1620"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<fp16, [1, 1024, 32, 128]> z_5_cast_fp16 = reshape(shape = var_1620, x = linear_17_cast_fp16)[name = string("z_5_cast_fp16")];
+            tensor<fp16, [32]> linear_18_bias_0_to_fp16 = const()[name = string("linear_18_bias_0_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(685678976)))];
+            tensor<fp16, [1, 1024, 32]> linear_18_cast_fp16 = linear(bias = linear_18_bias_0_to_fp16, weight = layers_2_linear_attn_in_proj_b_weight_cast_fp16, x = x_55_cast_fp16)[name = string("linear_18_cast_fp16")];
+            tensor<fp16, [1, 1024, 32]> beta_9_cast_fp16 = sigmoid(x = linear_18_cast_fp16)[name = string("beta_9_cast_fp16")];
+            tensor<fp16, [32]> linear_19_bias_0_to_fp16 = const()[name = string("linear_19_bias_0_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(685679104)))];
+            tensor<fp16, [1, 1024, 32]> linear_19_cast_fp16 = linear(bias = linear_19_bias_0_to_fp16, weight = layers_2_linear_attn_in_proj_a_weight_cast_fp16, x = x_55_cast_fp16)[name = string("linear_19_cast_fp16")];
+            tensor<fp16, [32]> layers_2_linear_attn_dt_bias_to_fp16 = const()[name = string("layers_2_linear_attn_dt_bias_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(685679232)))];
+            tensor<fp16, [1, 1024, 32]> var_1629_cast_fp16 = add(x = linear_19_cast_fp16, y = layers_2_linear_attn_dt_bias_to_fp16)[name = string("op_1629_cast_fp16")];
+            tensor<fp16, [1, 1024, 32]> var_1630_cast_fp16 = softplus(x = var_1629_cast_fp16)[name = string("op_1630_cast_fp16")];
+            tensor<fp16, [32]> var_1626_to_fp16 = const()[name = string("op_1626_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(685679360)))];
+            tensor<fp16, [1, 1024, 32]> g_9_cast_fp16 = mul(x = var_1626_to_fp16, y = var_1630_cast_fp16)[name = string("g_9_cast_fp16")];
+            tensor<fp16, [1, 1024, 16, 128]> var_1632_cast_fp16 = mul(x = q_21_cast_fp16, y = q_21_cast_fp16)[name = string("op_1632_cast_fp16")];
+            tensor<int32, [1]> var_1634_axes_0 = const()[name = string("op_1634_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_1634_keep_dims_0 = const()[name = string("op_1634_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 1024, 16, 1]> var_1634_cast_fp16 = reduce_sum(axes = var_1634_axes_0, keep_dims = var_1634_keep_dims_0, x = var_1632_cast_fp16)[name = string("op_1634_cast_fp16")];
+            fp16 var_1635_to_fp16 = const()[name = string("op_1635_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 1024, 16, 1]> var_1636_cast_fp16 = add(x = var_1634_cast_fp16, y = var_1635_to_fp16)[name = string("op_1636_cast_fp16")];
+            fp32 var_1637_epsilon_0 = const()[name = string("op_1637_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 1024, 16, 1]> var_1637_cast_fp16 = rsqrt(epsilon = var_1637_epsilon_0, x = var_1636_cast_fp16)[name = string("op_1637_cast_fp16")];
+            tensor<fp16, [1, 1024, 16, 128]> q_23_cast_fp16 = mul(x = q_21_cast_fp16, y = var_1637_cast_fp16)[name = string("q_23_cast_fp16")];
+            tensor<fp16, [1, 1024, 16, 128]> var_1639_cast_fp16 = mul(x = k_17_cast_fp16, y = k_17_cast_fp16)[name = string("op_1639_cast_fp16")];
+            tensor<int32, [1]> var_1641_axes_0 = const()[name = string("op_1641_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_1641_keep_dims_0 = const()[name = string("op_1641_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 1024, 16, 1]> var_1641_cast_fp16 = reduce_sum(axes = var_1641_axes_0, keep_dims = var_1641_keep_dims_0, x = var_1639_cast_fp16)[name = string("op_1641_cast_fp16")];
+            fp16 var_1642_to_fp16 = const()[name = string("op_1642_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 1024, 16, 1]> var_1643_cast_fp16 = add(x = var_1641_cast_fp16, y = var_1642_to_fp16)[name = string("op_1643_cast_fp16")];
+            fp32 var_1644_epsilon_0 = const()[name = string("op_1644_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 1024, 16, 1]> var_1644_cast_fp16 = rsqrt(epsilon = var_1644_epsilon_0, x = var_1643_cast_fp16)[name = string("op_1644_cast_fp16")];
+            tensor<fp16, [1, 1024, 16, 128]> k_19_cast_fp16 = mul(x = k_17_cast_fp16, y = var_1644_cast_fp16)[name = string("k_19_cast_fp16")];
+            tensor<int32, [1]> var_1649_axes_0 = const()[name = string("op_1649_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [1, 1024, 16, 1, 128]> var_1649_cast_fp16 = expand_dims(axes = var_1649_axes_0, x = q_23_cast_fp16)[name = string("op_1649_cast_fp16")];
+            tensor<int32, [5]> var_1651_reps_0 = const()[name = string("op_1651_reps_0"), val = tensor<int32, [5]>([1, 1, 1, 2, 1])];
+            tensor<fp16, [1, 1024, 16, 2, 128]> var_1651_cast_fp16 = tile(reps = var_1651_reps_0, x = var_1649_cast_fp16)[name = string("op_1651_cast_fp16")];
+            tensor<int32, [4]> var_1652 = const()[name = string("op_1652"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<fp16, [1, 1024, 32, 128]> q_25_cast_fp16 = reshape(shape = var_1652, x = var_1651_cast_fp16)[name = string("q_25_cast_fp16")];
+            tensor<int32, [1]> var_1657_axes_0 = const()[name = string("op_1657_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [1, 1024, 16, 1, 128]> var_1657_cast_fp16 = expand_dims(axes = var_1657_axes_0, x = k_19_cast_fp16)[name = string("op_1657_cast_fp16")];
+            tensor<int32, [5]> var_1659_reps_0 = const()[name = string("op_1659_reps_0"), val = tensor<int32, [5]>([1, 1, 1, 2, 1])];
+            tensor<fp16, [1, 1024, 16, 2, 128]> var_1659_cast_fp16 = tile(reps = var_1659_reps_0, x = var_1657_cast_fp16)[name = string("op_1659_cast_fp16")];
+            tensor<int32, [4]> var_1660 = const()[name = string("op_1660"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<fp16, [1, 1024, 32, 128]> k_21_cast_fp16 = reshape(shape = var_1660, x = var_1659_cast_fp16)[name = string("k_21_cast_fp16")];
+            fp16 var_1662_to_fp16 = const()[name = string("op_1662_to_fp16"), val = fp16(0x1.6ap-4)];
+            tensor<fp16, [1, 1024, 32, 128]> q_27_cast_fp16 = mul(x = q_25_cast_fp16, y = var_1662_to_fp16)[name = string("q_27_cast_fp16")];
+            tensor<int32, [4]> var_1664_begin_0 = const()[name = string("op_1664_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_1664_end_0 = const()[name = string("op_1664_end_0"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<bool, [4]> var_1664_end_mask_0 = const()[name = string("op_1664_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_1664_squeeze_mask_0 = const()[name = string("op_1664_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [1024, 32, 128]> var_1664_cast_fp16 = slice_by_index(begin = var_1664_begin_0, end = var_1664_end_0, end_mask = var_1664_end_mask_0, squeeze_mask = var_1664_squeeze_mask_0, x = q_27_cast_fp16)[name = string("op_1664_cast_fp16")];
+            tensor<int32, [3]> var_1665 = const()[name = string("op_1665"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_1667 = const()[name = string("op_1667"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<fp16, [32, 1024, 128]> var_1666_cast_fp16 = transpose(perm = var_1665, x = var_1664_cast_fp16)[name = string("transpose_37")];
+            tensor<fp16, [32, 16, 64, 128]> q_29_cast_fp16 = reshape(shape = var_1667, x = var_1666_cast_fp16)[name = string("q_29_cast_fp16")];
+            tensor<int32, [4]> var_1669_begin_0 = const()[name = string("op_1669_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_1669_end_0 = const()[name = string("op_1669_end_0"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<bool, [4]> var_1669_end_mask_0 = const()[name = string("op_1669_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_1669_squeeze_mask_0 = const()[name = string("op_1669_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [1024, 32, 128]> var_1669_cast_fp16 = slice_by_index(begin = var_1669_begin_0, end = var_1669_end_0, end_mask = var_1669_end_mask_0, squeeze_mask = var_1669_squeeze_mask_0, x = k_21_cast_fp16)[name = string("op_1669_cast_fp16")];
+            tensor<int32, [3]> var_1670 = const()[name = string("op_1670"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_1672 = const()[name = string("op_1672"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<fp16, [32, 1024, 128]> var_1671_cast_fp16 = transpose(perm = var_1670, x = var_1669_cast_fp16)[name = string("transpose_36")];
+            tensor<fp16, [32, 16, 64, 128]> k_23_cast_fp16 = reshape(shape = var_1672, x = var_1671_cast_fp16)[name = string("k_23_cast_fp16")];
+            tensor<int32, [4]> var_1674_begin_0 = const()[name = string("op_1674_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_1674_end_0 = const()[name = string("op_1674_end_0"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<bool, [4]> var_1674_end_mask_0 = const()[name = string("op_1674_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_1674_squeeze_mask_0 = const()[name = string("op_1674_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [1024, 32, 128]> var_1674_cast_fp16 = slice_by_index(begin = var_1674_begin_0, end = var_1674_end_0, end_mask = var_1674_end_mask_0, squeeze_mask = var_1674_squeeze_mask_0, x = v_9_cast_fp16)[name = string("op_1674_cast_fp16")];
+            tensor<int32, [3]> var_1675 = const()[name = string("op_1675"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_1677 = const()[name = string("op_1677"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<fp16, [32, 1024, 128]> var_1676_cast_fp16 = transpose(perm = var_1675, x = var_1674_cast_fp16)[name = string("transpose_35")];
+            tensor<fp16, [32, 16, 64, 128]> v_11_cast_fp16 = reshape(shape = var_1677, x = var_1676_cast_fp16)[name = string("v_11_cast_fp16")];
+            tensor<int32, [3]> var_1679_begin_0 = const()[name = string("op_1679_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_1679_end_0 = const()[name = string("op_1679_end_0"), val = tensor<int32, [3]>([1, 1024, 32])];
+            tensor<bool, [3]> var_1679_end_mask_0 = const()[name = string("op_1679_end_mask_0"), val = tensor<bool, [3]>([false, true, true])];
+            tensor<bool, [3]> var_1679_squeeze_mask_0 = const()[name = string("op_1679_squeeze_mask_0"), val = tensor<bool, [3]>([true, false, false])];
+            tensor<fp16, [1024, 32]> var_1679_cast_fp16 = slice_by_index(begin = var_1679_begin_0, end = var_1679_end_0, end_mask = var_1679_end_mask_0, squeeze_mask = var_1679_squeeze_mask_0, x = beta_9_cast_fp16)[name = string("op_1679_cast_fp16")];
+            tensor<int32, [2]> var_1680_perm_0 = const()[name = string("op_1680_perm_0"), val = tensor<int32, [2]>([1, 0])];
+            tensor<int32, [4]> var_1681 = const()[name = string("op_1681"), val = tensor<int32, [4]>([32, 16, 64, 1])];
+            tensor<fp16, [32, 1024]> var_1680_cast_fp16 = transpose(perm = var_1680_perm_0, x = var_1679_cast_fp16)[name = string("transpose_34")];
+            tensor<fp16, [32, 16, 64, 1]> beta_11_cast_fp16 = reshape(shape = var_1681, x = var_1680_cast_fp16)[name = string("beta_11_cast_fp16")];
+            tensor<int32, [3]> var_1683_begin_0 = const()[name = string("op_1683_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_1683_end_0 = const()[name = string("op_1683_end_0"), val = tensor<int32, [3]>([1, 1024, 32])];
+            tensor<bool, [3]> var_1683_end_mask_0 = const()[name = string("op_1683_end_mask_0"), val = tensor<bool, [3]>([false, true, true])];
+            tensor<bool, [3]> var_1683_squeeze_mask_0 = const()[name = string("op_1683_squeeze_mask_0"), val = tensor<bool, [3]>([true, false, false])];
+            tensor<fp16, [1024, 32]> var_1683_cast_fp16 = slice_by_index(begin = var_1683_begin_0, end = var_1683_end_0, end_mask = var_1683_end_mask_0, squeeze_mask = var_1683_squeeze_mask_0, x = g_9_cast_fp16)[name = string("op_1683_cast_fp16")];
+            tensor<int32, [2]> var_1684_perm_0 = const()[name = string("op_1684_perm_0"), val = tensor<int32, [2]>([1, 0])];
+            tensor<int32, [3]> var_1685 = const()[name = string("op_1685"), val = tensor<int32, [3]>([32, 16, 64])];
+            tensor<fp16, [32, 1024]> var_1684_cast_fp16 = transpose(perm = var_1684_perm_0, x = var_1683_cast_fp16)[name = string("transpose_33")];
+            tensor<fp16, [32, 16, 64]> g_11_cast_fp16 = reshape(shape = var_1685, x = var_1684_cast_fp16)[name = string("g_11_cast_fp16")];
+            tensor<int32, [1]> g_col_5_axes_0 = const()[name = string("g_col_5_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 16, 64, 1]> g_col_5_cast_fp16 = expand_dims(axes = g_col_5_axes_0, x = g_11_cast_fp16)[name = string("g_col_5_cast_fp16")];
+            bool var_1688_transpose_x_0 = const()[name = string("op_1688_transpose_x_0"), val = bool(false)];
+            bool var_1688_transpose_y_0 = const()[name = string("op_1688_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 16, 64, 1]> var_1688_cast_fp16 = matmul(transpose_x = var_1688_transpose_x_0, transpose_y = var_1688_transpose_y_0, x = layers_0_linear_attn_lower_incl_to_fp16, y = g_col_5_cast_fp16)[name = string("op_1688_cast_fp16")];
+            tensor<int32, [1]> cum_5_axes_0 = const()[name = string("cum_5_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 16, 64]> cum_5_cast_fp16 = squeeze(axes = cum_5_axes_0, x = var_1688_cast_fp16)[name = string("cum_5_cast_fp16")];
+            bool var_1690_transpose_x_0 = const()[name = string("op_1690_transpose_x_0"), val = bool(false)];
+            bool var_1690_transpose_y_0 = const()[name = string("op_1690_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 16, 64, 1]> var_1690_cast_fp16 = matmul(transpose_x = var_1690_transpose_x_0, transpose_y = var_1690_transpose_y_0, x = layers_0_linear_attn_suffix_sum_t_to_fp16, y = g_col_5_cast_fp16)[name = string("op_1690_cast_fp16")];
+            tensor<int32, [1]> to_end_5_axes_0 = const()[name = string("to_end_5_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 16, 64]> to_end_5_cast_fp16 = squeeze(axes = to_end_5_axes_0, x = var_1690_cast_fp16)[name = string("to_end_5_cast_fp16")];
+            bool var_1692_transpose_x_0 = const()[name = string("op_1692_transpose_x_0"), val = bool(false)];
+            bool var_1692_transpose_y_0 = const()[name = string("op_1692_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 16, 4096, 1]> var_1692_cast_fp16 = matmul(transpose_x = var_1692_transpose_x_0, transpose_y = var_1692_transpose_y_0, x = layers_0_linear_attn_pair_sum_t_to_fp16, y = g_col_5_cast_fp16)[name = string("op_1692_cast_fp16")];
+            tensor<int32, [4]> var_1693 = const()[name = string("op_1693"), val = tensor<int32, [4]>([32, 16, 64, 64])];
+            tensor<fp16, [32, 16, 64, 64]> pair_29_cast_fp16 = reshape(shape = var_1693, x = var_1692_cast_fp16)[name = string("pair_29_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 64]> var_1695_cast_fp16 = exp(x = pair_29_cast_fp16)[name = string("op_1695_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 64]> pair_decay_5_cast_fp16 = mul(x = var_1695_cast_fp16, y = layers_0_linear_attn_lower_incl_to_fp16)[name = string("pair_decay_5_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 128]> k_beta_5_cast_fp16 = mul(x = k_23_cast_fp16, y = beta_11_cast_fp16)[name = string("k_beta_5_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 128]> v_beta_5_cast_fp16 = mul(x = v_11_cast_fp16, y = beta_11_cast_fp16)[name = string("v_beta_5_cast_fp16")];
+            bool var_1700_transpose_x_1 = const()[name = string("op_1700_transpose_x_1"), val = bool(false)];
+            bool var_1700_transpose_y_1 = const()[name = string("op_1700_transpose_y_1"), val = bool(true)];
+            tensor<fp16, [32, 16, 64, 64]> var_1700_cast_fp16 = matmul(transpose_x = var_1700_transpose_x_1, transpose_y = var_1700_transpose_y_1, x = k_beta_5_cast_fp16, y = k_23_cast_fp16)[name = string("op_1700_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 64]> kkt_5_cast_fp16 = mul(x = var_1700_cast_fp16, y = pair_decay_5_cast_fp16)[name = string("kkt_5_cast_fp16")];
+            bool var_1703_transpose_x_1 = const()[name = string("op_1703_transpose_x_1"), val = bool(false)];
+            bool var_1703_transpose_y_1 = const()[name = string("op_1703_transpose_y_1"), val = bool(true)];
+            tensor<fp16, [32, 16, 64, 64]> var_1703_cast_fp16 = matmul(transpose_x = var_1703_transpose_x_1, transpose_y = var_1703_transpose_y_1, x = q_29_cast_fp16, y = k_23_cast_fp16)[name = string("op_1703_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 64]> attn_intra_5_cast_fp16 = mul(x = var_1703_cast_fp16, y = pair_decay_5_cast_fp16)[name = string("attn_intra_5_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 64]> var_1705_cast_fp16 = mul(x = kkt_5_cast_fp16, y = layers_0_linear_attn_strict_lower_to_fp16)[name = string("op_1705_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 64]> t_9_cast_fp16 = add(x = layers_0_linear_attn_eye_to_fp16, y = var_1705_cast_fp16)[name = string("t_9_cast_fp16")];
+            tensor<int32, [3]> var_1709 = const()[name = string("op_1709"), val = tensor<int32, [3]>([-1, 64, 64])];
+            tensor<fp16, [512, 64, 64]> t_11_cast_fp16 = reshape(shape = var_1709, x = t_9_cast_fp16)[name = string("t_11_cast_fp16")];
+            tensor<int32, [5]> var_1714 = const()[name = string("op_1714"), val = tensor<int32, [5]>([512, 32, 2, 32, 2])];
+            tensor<fp16, [512, 32, 2, 32, 2]> var_1715_cast_fp16 = reshape(shape = var_1714, x = t_11_cast_fp16)[name = string("op_1715_cast_fp16")];
+            tensor<fp16, [512, 32, 2, 32, 2]> var_1720_cast_fp16 = mul(x = var_1715_cast_fp16, y = var_213_to_fp16)[name = string("op_1720_cast_fp16")];
+            tensor<int32, [1]> blocks_25_axes_0 = const()[name = string("blocks_25_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_25_keep_dims_0 = const()[name = string("blocks_25_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 32, 2, 2]> blocks_25_cast_fp16 = reduce_sum(axes = blocks_25_axes_0, keep_dims = blocks_25_keep_dims_0, x = var_1720_cast_fp16)[name = string("blocks_25_cast_fp16")];
+            tensor<int32, [4]> var_1723_begin_0 = const()[name = string("op_1723_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
+            tensor<int32, [4]> var_1723_end_0 = const()[name = string("op_1723_end_0"), val = tensor<int32, [4]>([512, 32, 2, 2])];
+            tensor<bool, [4]> var_1723_end_mask_0 = const()[name = string("op_1723_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 32, 1, 2]> var_1723_cast_fp16 = slice_by_index(begin = var_1723_begin_0, end = var_1723_end_0, end_mask = var_1723_end_mask_0, x = blocks_25_cast_fp16)[name = string("op_1723_cast_fp16")];
+            tensor<int32, [4]> x_57_begin_0 = const()[name = string("x_57_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_57_end_0 = const()[name = string("x_57_end_0"), val = tensor<int32, [4]>([512, 32, 1, 1])];
+            tensor<bool, [4]> x_57_end_mask_0 = const()[name = string("x_57_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 32, 1, 1]> x_57_cast_fp16 = slice_by_index(begin = x_57_begin_0, end = x_57_end_0, end_mask = x_57_end_mask_0, x = var_1723_cast_fp16)[name = string("x_57_cast_fp16")];
+            bool var_1733_transpose_x_0 = const()[name = string("op_1733_transpose_x_0"), val = bool(false)];
+            bool var_1733_transpose_y_0 = const()[name = string("op_1733_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 32, 1, 1]> var_1733_cast_fp16 = matmul(transpose_x = var_1733_transpose_x_0, transpose_y = var_1733_transpose_y_0, x = x_57_cast_fp16, y = var_221_to_fp16)[name = string("op_1733_cast_fp16")];
+            bool var_1734_transpose_x_0 = const()[name = string("op_1734_transpose_x_0"), val = bool(false)];
+            bool var_1734_transpose_y_0 = const()[name = string("op_1734_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 32, 1, 1]> var_1734_cast_fp16 = matmul(transpose_x = var_1734_transpose_x_0, transpose_y = var_1734_transpose_y_0, x = var_221_to_fp16, y = var_1733_cast_fp16)[name = string("op_1734_cast_fp16")];
+            fp16 const_30_promoted_to_fp16 = const()[name = string("const_30_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 32, 1, 1]> lower_25_cast_fp16 = mul(x = var_1734_cast_fp16, y = const_30_promoted_to_fp16)[name = string("lower_25_cast_fp16")];
+            bool bottom_25_interleave_0 = const()[name = string("bottom_25_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 32, 1, 2]> bottom_25_cast_fp16 = concat(axis = var_1555, interleave = bottom_25_interleave_0, values = (lower_25_cast_fp16, var_221_to_fp16))[name = string("bottom_25_cast_fp16")];
+            bool inv_35_interleave_0 = const()[name = string("inv_35_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 32, 2, 2]> inv_35_cast_fp16 = concat(axis = var_1540, interleave = inv_35_interleave_0, values = (top_1_to_fp16, bottom_25_cast_fp16))[name = string("inv_35_cast_fp16")];
+            tensor<int32, [5]> var_1743 = const()[name = string("op_1743"), val = tensor<int32, [5]>([512, 16, 4, 16, 4])];
+            tensor<fp16, [512, 16, 4, 16, 4]> var_1744_cast_fp16 = reshape(shape = var_1743, x = t_11_cast_fp16)[name = string("op_1744_cast_fp16")];
+            tensor<fp16, [512, 16, 4, 16, 4]> var_1749_cast_fp16 = mul(x = var_1744_cast_fp16, y = var_242_to_fp16)[name = string("op_1749_cast_fp16")];
+            tensor<int32, [1]> blocks_27_axes_0 = const()[name = string("blocks_27_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_27_keep_dims_0 = const()[name = string("blocks_27_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 4, 4]> blocks_27_cast_fp16 = reduce_sum(axes = blocks_27_axes_0, keep_dims = blocks_27_keep_dims_0, x = var_1749_cast_fp16)[name = string("blocks_27_cast_fp16")];
+            tensor<int32, [4]> var_1752_begin_0 = const()[name = string("op_1752_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
+            tensor<int32, [4]> var_1752_end_0 = const()[name = string("op_1752_end_0"), val = tensor<int32, [4]>([512, 16, 4, 4])];
+            tensor<bool, [4]> var_1752_end_mask_0 = const()[name = string("op_1752_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 16, 2, 4]> var_1752_cast_fp16 = slice_by_index(begin = var_1752_begin_0, end = var_1752_end_0, end_mask = var_1752_end_mask_0, x = blocks_27_cast_fp16)[name = string("op_1752_cast_fp16")];
+            tensor<int32, [4]> x_59_begin_0 = const()[name = string("x_59_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_59_end_0 = const()[name = string("x_59_end_0"), val = tensor<int32, [4]>([512, 16, 2, 2])];
+            tensor<bool, [4]> x_59_end_mask_0 = const()[name = string("x_59_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 16, 2, 2]> x_59_cast_fp16 = slice_by_index(begin = x_59_begin_0, end = x_59_end_0, end_mask = x_59_end_mask_0, x = var_1752_cast_fp16)[name = string("x_59_cast_fp16")];
+            tensor<int32, [5]> var_1754 = const()[name = string("op_1754"), val = tensor<int32, [5]>([512, 16, 2, 2, 2])];
+            tensor<fp16, [512, 16, 2, 2, 2]> pair_33_cast_fp16 = reshape(shape = var_1754, x = inv_35_cast_fp16)[name = string("pair_33_cast_fp16")];
+            tensor<int32, [5]> var_1756_begin_0 = const()[name = string("op_1756_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_1756_end_0 = const()[name = string("op_1756_end_0"), val = tensor<int32, [5]>([512, 16, 1, 2, 2])];
+            tensor<bool, [5]> var_1756_end_mask_0 = const()[name = string("op_1756_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_1756_squeeze_mask_0 = const()[name = string("op_1756_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 16, 2, 2]> var_1756_cast_fp16 = slice_by_index(begin = var_1756_begin_0, end = var_1756_end_0, end_mask = var_1756_end_mask_0, squeeze_mask = var_1756_squeeze_mask_0, x = pair_33_cast_fp16)[name = string("op_1756_cast_fp16")];
+            tensor<int32, [5]> var_1759_begin_0 = const()[name = string("op_1759_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_1759_end_0 = const()[name = string("op_1759_end_0"), val = tensor<int32, [5]>([512, 16, 2, 2, 2])];
+            tensor<bool, [5]> var_1759_end_mask_0 = const()[name = string("op_1759_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_1759_squeeze_mask_0 = const()[name = string("op_1759_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 16, 2, 2]> var_1759_cast_fp16 = slice_by_index(begin = var_1759_begin_0, end = var_1759_end_0, end_mask = var_1759_end_mask_0, squeeze_mask = var_1759_squeeze_mask_0, x = pair_33_cast_fp16)[name = string("op_1759_cast_fp16")];
+            bool var_1762_transpose_x_0 = const()[name = string("op_1762_transpose_x_0"), val = bool(false)];
+            bool var_1762_transpose_y_0 = const()[name = string("op_1762_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 2, 2]> var_1762_cast_fp16 = matmul(transpose_x = var_1762_transpose_x_0, transpose_y = var_1762_transpose_y_0, x = x_59_cast_fp16, y = var_1756_cast_fp16)[name = string("op_1762_cast_fp16")];
+            bool var_1763_transpose_x_0 = const()[name = string("op_1763_transpose_x_0"), val = bool(false)];
+            bool var_1763_transpose_y_0 = const()[name = string("op_1763_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 2, 2]> var_1763_cast_fp16 = matmul(transpose_x = var_1763_transpose_x_0, transpose_y = var_1763_transpose_y_0, x = var_1759_cast_fp16, y = var_1762_cast_fp16)[name = string("op_1763_cast_fp16")];
+            fp16 const_31_promoted_to_fp16 = const()[name = string("const_31_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 16, 2, 2]> lower_27_cast_fp16 = mul(x = var_1763_cast_fp16, y = const_31_promoted_to_fp16)[name = string("lower_27_cast_fp16")];
+            bool top_27_interleave_0 = const()[name = string("top_27_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 2, 4]> top_27_cast_fp16 = concat(axis = var_1555, interleave = top_27_interleave_0, values = (var_1756_cast_fp16, var_259_to_fp16))[name = string("top_27_cast_fp16")];
+            bool bottom_27_interleave_0 = const()[name = string("bottom_27_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 2, 4]> bottom_27_cast_fp16 = concat(axis = var_1555, interleave = bottom_27_interleave_0, values = (lower_27_cast_fp16, var_1759_cast_fp16))[name = string("bottom_27_cast_fp16")];
+            bool inv_37_interleave_0 = const()[name = string("inv_37_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 4, 4]> inv_37_cast_fp16 = concat(axis = var_1540, interleave = inv_37_interleave_0, values = (top_27_cast_fp16, bottom_27_cast_fp16))[name = string("inv_37_cast_fp16")];
+            tensor<int32, [5]> var_1772 = const()[name = string("op_1772"), val = tensor<int32, [5]>([512, 8, 8, 8, 8])];
+            tensor<fp16, [512, 8, 8, 8, 8]> var_1773_cast_fp16 = reshape(shape = var_1772, x = t_11_cast_fp16)[name = string("op_1773_cast_fp16")];
+            tensor<fp16, [8, 1, 8, 1]> var_1777_to_fp16 = const()[name = string("op_1777_to_fp16"), val = tensor<fp16, [8, 1, 8, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(685679488)))];
+            tensor<fp16, [512, 8, 8, 8, 8]> var_1778_cast_fp16 = mul(x = var_1773_cast_fp16, y = var_1777_to_fp16)[name = string("op_1778_cast_fp16")];
+            tensor<int32, [1]> blocks_29_axes_0 = const()[name = string("blocks_29_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_29_keep_dims_0 = const()[name = string("blocks_29_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 8, 8]> blocks_29_cast_fp16 = reduce_sum(axes = blocks_29_axes_0, keep_dims = blocks_29_keep_dims_0, x = var_1778_cast_fp16)[name = string("blocks_29_cast_fp16")];
+            tensor<int32, [4]> var_1781_begin_0 = const()[name = string("op_1781_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
+            tensor<int32, [4]> var_1781_end_0 = const()[name = string("op_1781_end_0"), val = tensor<int32, [4]>([512, 8, 8, 8])];
+            tensor<bool, [4]> var_1781_end_mask_0 = const()[name = string("op_1781_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 8, 4, 8]> var_1781_cast_fp16 = slice_by_index(begin = var_1781_begin_0, end = var_1781_end_0, end_mask = var_1781_end_mask_0, x = blocks_29_cast_fp16)[name = string("op_1781_cast_fp16")];
+            tensor<int32, [4]> x_61_begin_0 = const()[name = string("x_61_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_61_end_0 = const()[name = string("x_61_end_0"), val = tensor<int32, [4]>([512, 8, 4, 4])];
+            tensor<bool, [4]> x_61_end_mask_0 = const()[name = string("x_61_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 8, 4, 4]> x_61_cast_fp16 = slice_by_index(begin = x_61_begin_0, end = x_61_end_0, end_mask = x_61_end_mask_0, x = var_1781_cast_fp16)[name = string("x_61_cast_fp16")];
+            tensor<int32, [5]> var_1783 = const()[name = string("op_1783"), val = tensor<int32, [5]>([512, 8, 2, 4, 4])];
+            tensor<fp16, [512, 8, 2, 4, 4]> pair_35_cast_fp16 = reshape(shape = var_1783, x = inv_37_cast_fp16)[name = string("pair_35_cast_fp16")];
+            tensor<int32, [5]> var_1785_begin_0 = const()[name = string("op_1785_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_1785_end_0 = const()[name = string("op_1785_end_0"), val = tensor<int32, [5]>([512, 8, 1, 4, 4])];
+            tensor<bool, [5]> var_1785_end_mask_0 = const()[name = string("op_1785_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_1785_squeeze_mask_0 = const()[name = string("op_1785_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 8, 4, 4]> var_1785_cast_fp16 = slice_by_index(begin = var_1785_begin_0, end = var_1785_end_0, end_mask = var_1785_end_mask_0, squeeze_mask = var_1785_squeeze_mask_0, x = pair_35_cast_fp16)[name = string("op_1785_cast_fp16")];
+            tensor<int32, [5]> var_1788_begin_0 = const()[name = string("op_1788_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_1788_end_0 = const()[name = string("op_1788_end_0"), val = tensor<int32, [5]>([512, 8, 2, 4, 4])];
+            tensor<bool, [5]> var_1788_end_mask_0 = const()[name = string("op_1788_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_1788_squeeze_mask_0 = const()[name = string("op_1788_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 8, 4, 4]> var_1788_cast_fp16 = slice_by_index(begin = var_1788_begin_0, end = var_1788_end_0, end_mask = var_1788_end_mask_0, squeeze_mask = var_1788_squeeze_mask_0, x = pair_35_cast_fp16)[name = string("op_1788_cast_fp16")];
+            bool var_1791_transpose_x_0 = const()[name = string("op_1791_transpose_x_0"), val = bool(false)];
+            bool var_1791_transpose_y_0 = const()[name = string("op_1791_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 4, 4]> var_1791_cast_fp16 = matmul(transpose_x = var_1791_transpose_x_0, transpose_y = var_1791_transpose_y_0, x = x_61_cast_fp16, y = var_1785_cast_fp16)[name = string("op_1791_cast_fp16")];
+            bool var_1792_transpose_x_0 = const()[name = string("op_1792_transpose_x_0"), val = bool(false)];
+            bool var_1792_transpose_y_0 = const()[name = string("op_1792_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 4, 4]> var_1792_cast_fp16 = matmul(transpose_x = var_1792_transpose_x_0, transpose_y = var_1792_transpose_y_0, x = var_1788_cast_fp16, y = var_1791_cast_fp16)[name = string("op_1792_cast_fp16")];
+            fp16 const_32_promoted_to_fp16 = const()[name = string("const_32_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 8, 4, 4]> lower_29_cast_fp16 = mul(x = var_1792_cast_fp16, y = const_32_promoted_to_fp16)[name = string("lower_29_cast_fp16")];
+            bool top_29_interleave_0 = const()[name = string("top_29_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 4, 8]> top_29_cast_fp16 = concat(axis = var_1555, interleave = top_29_interleave_0, values = (var_1785_cast_fp16, var_288_to_fp16))[name = string("top_29_cast_fp16")];
+            bool bottom_29_interleave_0 = const()[name = string("bottom_29_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 4, 8]> bottom_29_cast_fp16 = concat(axis = var_1555, interleave = bottom_29_interleave_0, values = (lower_29_cast_fp16, var_1788_cast_fp16))[name = string("bottom_29_cast_fp16")];
+            bool inv_39_interleave_0 = const()[name = string("inv_39_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 8, 8]> inv_39_cast_fp16 = concat(axis = var_1540, interleave = inv_39_interleave_0, values = (top_29_cast_fp16, bottom_29_cast_fp16))[name = string("inv_39_cast_fp16")];
+            tensor<int32, [5]> var_1801 = const()[name = string("op_1801"), val = tensor<int32, [5]>([512, 4, 16, 4, 16])];
+            tensor<fp16, [512, 4, 16, 4, 16]> var_1802_cast_fp16 = reshape(shape = var_1801, x = t_11_cast_fp16)[name = string("op_1802_cast_fp16")];
+            tensor<fp16, [4, 1, 4, 1]> var_1806_to_fp16 = const()[name = string("op_1806_to_fp16"), val = tensor<fp16, [4, 1, 4, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(685679680)))];
+            tensor<fp16, [512, 4, 16, 4, 16]> var_1807_cast_fp16 = mul(x = var_1802_cast_fp16, y = var_1806_to_fp16)[name = string("op_1807_cast_fp16")];
+            tensor<int32, [1]> blocks_31_axes_0 = const()[name = string("blocks_31_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_31_keep_dims_0 = const()[name = string("blocks_31_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 16, 16]> blocks_31_cast_fp16 = reduce_sum(axes = blocks_31_axes_0, keep_dims = blocks_31_keep_dims_0, x = var_1807_cast_fp16)[name = string("blocks_31_cast_fp16")];
+            tensor<int32, [4]> var_1810_begin_0 = const()[name = string("op_1810_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
+            tensor<int32, [4]> var_1810_end_0 = const()[name = string("op_1810_end_0"), val = tensor<int32, [4]>([512, 4, 16, 16])];
+            tensor<bool, [4]> var_1810_end_mask_0 = const()[name = string("op_1810_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 4, 8, 16]> var_1810_cast_fp16 = slice_by_index(begin = var_1810_begin_0, end = var_1810_end_0, end_mask = var_1810_end_mask_0, x = blocks_31_cast_fp16)[name = string("op_1810_cast_fp16")];
+            tensor<int32, [4]> x_63_begin_0 = const()[name = string("x_63_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_63_end_0 = const()[name = string("x_63_end_0"), val = tensor<int32, [4]>([512, 4, 8, 8])];
+            tensor<bool, [4]> x_63_end_mask_0 = const()[name = string("x_63_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 4, 8, 8]> x_63_cast_fp16 = slice_by_index(begin = x_63_begin_0, end = x_63_end_0, end_mask = x_63_end_mask_0, x = var_1810_cast_fp16)[name = string("x_63_cast_fp16")];
+            tensor<int32, [5]> var_1812 = const()[name = string("op_1812"), val = tensor<int32, [5]>([512, 4, 2, 8, 8])];
+            tensor<fp16, [512, 4, 2, 8, 8]> pair_37_cast_fp16 = reshape(shape = var_1812, x = inv_39_cast_fp16)[name = string("pair_37_cast_fp16")];
+            tensor<int32, [5]> var_1814_begin_0 = const()[name = string("op_1814_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_1814_end_0 = const()[name = string("op_1814_end_0"), val = tensor<int32, [5]>([512, 4, 1, 8, 8])];
+            tensor<bool, [5]> var_1814_end_mask_0 = const()[name = string("op_1814_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_1814_squeeze_mask_0 = const()[name = string("op_1814_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 4, 8, 8]> var_1814_cast_fp16 = slice_by_index(begin = var_1814_begin_0, end = var_1814_end_0, end_mask = var_1814_end_mask_0, squeeze_mask = var_1814_squeeze_mask_0, x = pair_37_cast_fp16)[name = string("op_1814_cast_fp16")];
+            tensor<int32, [5]> var_1817_begin_0 = const()[name = string("op_1817_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_1817_end_0 = const()[name = string("op_1817_end_0"), val = tensor<int32, [5]>([512, 4, 2, 8, 8])];
+            tensor<bool, [5]> var_1817_end_mask_0 = const()[name = string("op_1817_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_1817_squeeze_mask_0 = const()[name = string("op_1817_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 4, 8, 8]> var_1817_cast_fp16 = slice_by_index(begin = var_1817_begin_0, end = var_1817_end_0, end_mask = var_1817_end_mask_0, squeeze_mask = var_1817_squeeze_mask_0, x = pair_37_cast_fp16)[name = string("op_1817_cast_fp16")];
+            bool var_1820_transpose_x_0 = const()[name = string("op_1820_transpose_x_0"), val = bool(false)];
+            bool var_1820_transpose_y_0 = const()[name = string("op_1820_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 8, 8]> var_1820_cast_fp16 = matmul(transpose_x = var_1820_transpose_x_0, transpose_y = var_1820_transpose_y_0, x = x_63_cast_fp16, y = var_1814_cast_fp16)[name = string("op_1820_cast_fp16")];
+            bool var_1821_transpose_x_0 = const()[name = string("op_1821_transpose_x_0"), val = bool(false)];
+            bool var_1821_transpose_y_0 = const()[name = string("op_1821_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 8, 8]> var_1821_cast_fp16 = matmul(transpose_x = var_1821_transpose_x_0, transpose_y = var_1821_transpose_y_0, x = var_1817_cast_fp16, y = var_1820_cast_fp16)[name = string("op_1821_cast_fp16")];
+            fp16 const_33_promoted_to_fp16 = const()[name = string("const_33_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 4, 8, 8]> lower_31_cast_fp16 = mul(x = var_1821_cast_fp16, y = const_33_promoted_to_fp16)[name = string("lower_31_cast_fp16")];
+            bool top_31_interleave_0 = const()[name = string("top_31_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 8, 16]> top_31_cast_fp16 = concat(axis = var_1555, interleave = top_31_interleave_0, values = (var_1814_cast_fp16, var_317_to_fp16))[name = string("top_31_cast_fp16")];
+            bool bottom_31_interleave_0 = const()[name = string("bottom_31_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 8, 16]> bottom_31_cast_fp16 = concat(axis = var_1555, interleave = bottom_31_interleave_0, values = (lower_31_cast_fp16, var_1817_cast_fp16))[name = string("bottom_31_cast_fp16")];
+            bool inv_41_interleave_0 = const()[name = string("inv_41_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 16, 16]> inv_41_cast_fp16 = concat(axis = var_1540, interleave = inv_41_interleave_0, values = (top_31_cast_fp16, bottom_31_cast_fp16))[name = string("inv_41_cast_fp16")];
+            tensor<int32, [5]> var_1830 = const()[name = string("op_1830"), val = tensor<int32, [5]>([512, 2, 32, 2, 32])];
+            tensor<fp16, [512, 2, 32, 2, 32]> var_1831_cast_fp16 = reshape(shape = var_1830, x = t_11_cast_fp16)[name = string("op_1831_cast_fp16")];
+            tensor<fp16, [2, 1, 2, 1]> var_1835_to_fp16 = const()[name = string("op_1835_to_fp16"), val = tensor<fp16, [2, 1, 2, 1]>([[[[0x1p+0], [0x0p+0]]], [[[0x0p+0], [0x1p+0]]]])];
+            tensor<fp16, [512, 2, 32, 2, 32]> var_1836_cast_fp16 = mul(x = var_1831_cast_fp16, y = var_1835_to_fp16)[name = string("op_1836_cast_fp16")];
+            tensor<int32, [1]> blocks_33_axes_0 = const()[name = string("blocks_33_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_33_keep_dims_0 = const()[name = string("blocks_33_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 32, 32]> blocks_33_cast_fp16 = reduce_sum(axes = blocks_33_axes_0, keep_dims = blocks_33_keep_dims_0, x = var_1836_cast_fp16)[name = string("blocks_33_cast_fp16")];
+            tensor<int32, [4]> var_1839_begin_0 = const()[name = string("op_1839_begin_0"), val = tensor<int32, [4]>([0, 0, 16, 0])];
+            tensor<int32, [4]> var_1839_end_0 = const()[name = string("op_1839_end_0"), val = tensor<int32, [4]>([512, 2, 32, 32])];
+            tensor<bool, [4]> var_1839_end_mask_0 = const()[name = string("op_1839_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 2, 16, 32]> var_1839_cast_fp16 = slice_by_index(begin = var_1839_begin_0, end = var_1839_end_0, end_mask = var_1839_end_mask_0, x = blocks_33_cast_fp16)[name = string("op_1839_cast_fp16")];
+            tensor<int32, [4]> x_65_begin_0 = const()[name = string("x_65_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_65_end_0 = const()[name = string("x_65_end_0"), val = tensor<int32, [4]>([512, 2, 16, 16])];
+            tensor<bool, [4]> x_65_end_mask_0 = const()[name = string("x_65_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 2, 16, 16]> x_65_cast_fp16 = slice_by_index(begin = x_65_begin_0, end = x_65_end_0, end_mask = x_65_end_mask_0, x = var_1839_cast_fp16)[name = string("x_65_cast_fp16")];
+            tensor<int32, [5]> var_1841 = const()[name = string("op_1841"), val = tensor<int32, [5]>([512, 2, 2, 16, 16])];
+            tensor<fp16, [512, 2, 2, 16, 16]> pair_39_cast_fp16 = reshape(shape = var_1841, x = inv_41_cast_fp16)[name = string("pair_39_cast_fp16")];
+            tensor<int32, [5]> var_1843_begin_0 = const()[name = string("op_1843_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_1843_end_0 = const()[name = string("op_1843_end_0"), val = tensor<int32, [5]>([512, 2, 1, 16, 16])];
+            tensor<bool, [5]> var_1843_end_mask_0 = const()[name = string("op_1843_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_1843_squeeze_mask_0 = const()[name = string("op_1843_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 2, 16, 16]> var_1843_cast_fp16 = slice_by_index(begin = var_1843_begin_0, end = var_1843_end_0, end_mask = var_1843_end_mask_0, squeeze_mask = var_1843_squeeze_mask_0, x = pair_39_cast_fp16)[name = string("op_1843_cast_fp16")];
+            tensor<int32, [5]> var_1846_begin_0 = const()[name = string("op_1846_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_1846_end_0 = const()[name = string("op_1846_end_0"), val = tensor<int32, [5]>([512, 2, 2, 16, 16])];
+            tensor<bool, [5]> var_1846_end_mask_0 = const()[name = string("op_1846_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_1846_squeeze_mask_0 = const()[name = string("op_1846_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 2, 16, 16]> var_1846_cast_fp16 = slice_by_index(begin = var_1846_begin_0, end = var_1846_end_0, end_mask = var_1846_end_mask_0, squeeze_mask = var_1846_squeeze_mask_0, x = pair_39_cast_fp16)[name = string("op_1846_cast_fp16")];
+            bool var_1849_transpose_x_0 = const()[name = string("op_1849_transpose_x_0"), val = bool(false)];
+            bool var_1849_transpose_y_0 = const()[name = string("op_1849_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 16, 16]> var_1849_cast_fp16 = matmul(transpose_x = var_1849_transpose_x_0, transpose_y = var_1849_transpose_y_0, x = x_65_cast_fp16, y = var_1843_cast_fp16)[name = string("op_1849_cast_fp16")];
+            bool var_1850_transpose_x_0 = const()[name = string("op_1850_transpose_x_0"), val = bool(false)];
+            bool var_1850_transpose_y_0 = const()[name = string("op_1850_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 16, 16]> var_1850_cast_fp16 = matmul(transpose_x = var_1850_transpose_x_0, transpose_y = var_1850_transpose_y_0, x = var_1846_cast_fp16, y = var_1849_cast_fp16)[name = string("op_1850_cast_fp16")];
+            fp16 const_34_promoted_to_fp16 = const()[name = string("const_34_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 2, 16, 16]> lower_33_cast_fp16 = mul(x = var_1850_cast_fp16, y = const_34_promoted_to_fp16)[name = string("lower_33_cast_fp16")];
+            bool top_33_interleave_0 = const()[name = string("top_33_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 16, 32]> top_33_cast_fp16 = concat(axis = var_1555, interleave = top_33_interleave_0, values = (var_1843_cast_fp16, var_346_to_fp16))[name = string("top_33_cast_fp16")];
+            bool bottom_33_interleave_0 = const()[name = string("bottom_33_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 16, 32]> bottom_33_cast_fp16 = concat(axis = var_1555, interleave = bottom_33_interleave_0, values = (lower_33_cast_fp16, var_1846_cast_fp16))[name = string("bottom_33_cast_fp16")];
+            bool inv_43_interleave_0 = const()[name = string("inv_43_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 32, 32]> inv_43_cast_fp16 = concat(axis = var_1540, interleave = inv_43_interleave_0, values = (top_33_cast_fp16, bottom_33_cast_fp16))[name = string("inv_43_cast_fp16")];
+            tensor<int32, [5]> var_1859 = const()[name = string("op_1859"), val = tensor<int32, [5]>([512, 1, 64, 1, 64])];
+            tensor<fp16, [512, 1, 64, 1, 64]> var_1860_cast_fp16 = reshape(shape = var_1859, x = t_11_cast_fp16)[name = string("op_1860_cast_fp16")];
+            tensor<int32, [1]> blocks_35_axes_0 = const()[name = string("blocks_35_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_35_keep_dims_0 = const()[name = string("blocks_35_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 64, 64]> blocks_35_cast_fp16 = reduce_sum(axes = blocks_35_axes_0, keep_dims = blocks_35_keep_dims_0, x = var_1860_cast_fp16)[name = string("blocks_35_cast_fp16")];
+            tensor<int32, [4]> var_1868_begin_0 = const()[name = string("op_1868_begin_0"), val = tensor<int32, [4]>([0, 0, 32, 0])];
+            tensor<int32, [4]> var_1868_end_0 = const()[name = string("op_1868_end_0"), val = tensor<int32, [4]>([512, 1, 64, 64])];
+            tensor<bool, [4]> var_1868_end_mask_0 = const()[name = string("op_1868_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 1, 32, 64]> var_1868_cast_fp16 = slice_by_index(begin = var_1868_begin_0, end = var_1868_end_0, end_mask = var_1868_end_mask_0, x = blocks_35_cast_fp16)[name = string("op_1868_cast_fp16")];
+            tensor<int32, [4]> x_67_begin_0 = const()[name = string("x_67_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_67_end_0 = const()[name = string("x_67_end_0"), val = tensor<int32, [4]>([512, 1, 32, 32])];
+            tensor<bool, [4]> x_67_end_mask_0 = const()[name = string("x_67_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 1, 32, 32]> x_67_cast_fp16 = slice_by_index(begin = x_67_begin_0, end = x_67_end_0, end_mask = x_67_end_mask_0, x = var_1868_cast_fp16)[name = string("x_67_cast_fp16")];
+            tensor<int32, [5]> var_1870 = const()[name = string("op_1870"), val = tensor<int32, [5]>([512, 1, 2, 32, 32])];
+            tensor<fp16, [512, 1, 2, 32, 32]> pair_41_cast_fp16 = reshape(shape = var_1870, x = inv_43_cast_fp16)[name = string("pair_41_cast_fp16")];
+            tensor<int32, [5]> var_1872_begin_0 = const()[name = string("op_1872_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_1872_end_0 = const()[name = string("op_1872_end_0"), val = tensor<int32, [5]>([512, 1, 1, 32, 32])];
+            tensor<bool, [5]> var_1872_end_mask_0 = const()[name = string("op_1872_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_1872_squeeze_mask_0 = const()[name = string("op_1872_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 1, 32, 32]> var_1872_cast_fp16 = slice_by_index(begin = var_1872_begin_0, end = var_1872_end_0, end_mask = var_1872_end_mask_0, squeeze_mask = var_1872_squeeze_mask_0, x = pair_41_cast_fp16)[name = string("op_1872_cast_fp16")];
+            tensor<int32, [5]> var_1875_begin_0 = const()[name = string("op_1875_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_1875_end_0 = const()[name = string("op_1875_end_0"), val = tensor<int32, [5]>([512, 1, 2, 32, 32])];
+            tensor<bool, [5]> var_1875_end_mask_0 = const()[name = string("op_1875_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_1875_squeeze_mask_0 = const()[name = string("op_1875_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 1, 32, 32]> var_1875_cast_fp16 = slice_by_index(begin = var_1875_begin_0, end = var_1875_end_0, end_mask = var_1875_end_mask_0, squeeze_mask = var_1875_squeeze_mask_0, x = pair_41_cast_fp16)[name = string("op_1875_cast_fp16")];
+            bool var_1878_transpose_x_0 = const()[name = string("op_1878_transpose_x_0"), val = bool(false)];
+            bool var_1878_transpose_y_0 = const()[name = string("op_1878_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 32, 32]> var_1878_cast_fp16 = matmul(transpose_x = var_1878_transpose_x_0, transpose_y = var_1878_transpose_y_0, x = x_67_cast_fp16, y = var_1872_cast_fp16)[name = string("op_1878_cast_fp16")];
+            bool var_1879_transpose_x_0 = const()[name = string("op_1879_transpose_x_0"), val = bool(false)];
+            bool var_1879_transpose_y_0 = const()[name = string("op_1879_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 32, 32]> var_1879_cast_fp16 = matmul(transpose_x = var_1879_transpose_x_0, transpose_y = var_1879_transpose_y_0, x = var_1875_cast_fp16, y = var_1878_cast_fp16)[name = string("op_1879_cast_fp16")];
+            fp16 const_35_promoted_to_fp16 = const()[name = string("const_35_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 1, 32, 32]> lower_35_cast_fp16 = mul(x = var_1879_cast_fp16, y = const_35_promoted_to_fp16)[name = string("lower_35_cast_fp16")];
+            bool top_35_interleave_0 = const()[name = string("top_35_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 32, 64]> top_35_cast_fp16 = concat(axis = var_1555, interleave = top_35_interleave_0, values = (var_1872_cast_fp16, var_375_to_fp16))[name = string("top_35_cast_fp16")];
+            bool bottom_35_interleave_0 = const()[name = string("bottom_35_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 32, 64]> bottom_35_cast_fp16 = concat(axis = var_1555, interleave = bottom_35_interleave_0, values = (lower_35_cast_fp16, var_1875_cast_fp16))[name = string("bottom_35_cast_fp16")];
+            bool inv_45_interleave_0 = const()[name = string("inv_45_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 64, 64]> inv_45_cast_fp16 = concat(axis = var_1540, interleave = inv_45_interleave_0, values = (top_35_cast_fp16, bottom_35_cast_fp16))[name = string("inv_45_cast_fp16")];
+            tensor<int32, [4]> var_1888 = const()[name = string("op_1888"), val = tensor<int32, [4]>([32, 16, 64, 64])];
+            tensor<fp16, [32, 16, 64, 64]> inv_47_cast_fp16 = reshape(shape = var_1888, x = inv_45_cast_fp16)[name = string("inv_47_cast_fp16")];
+            bool new_v_5_transpose_x_0 = const()[name = string("new_v_5_transpose_x_0"), val = bool(false)];
+            bool new_v_5_transpose_y_0 = const()[name = string("new_v_5_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 16, 64, 128]> new_v_5_cast_fp16 = matmul(transpose_x = new_v_5_transpose_x_0, transpose_y = new_v_5_transpose_y_0, x = inv_47_cast_fp16, y = v_beta_5_cast_fp16)[name = string("new_v_5_cast_fp16")];
+            tensor<fp16, [32, 16, 64]> var_1891_cast_fp16 = exp(x = cum_5_cast_fp16)[name = string("op_1891_cast_fp16")];
+            tensor<int32, [1]> var_1892_axes_0 = const()[name = string("op_1892_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 16, 64, 1]> var_1892_cast_fp16 = expand_dims(axes = var_1892_axes_0, x = var_1891_cast_fp16)[name = string("op_1892_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 128]> var_1893_cast_fp16 = mul(x = k_beta_5_cast_fp16, y = var_1892_cast_fp16)[name = string("op_1893_cast_fp16")];
+            bool k_cumdecay_5_transpose_x_0 = const()[name = string("k_cumdecay_5_transpose_x_0"), val = bool(false)];
+            bool k_cumdecay_5_transpose_y_0 = const()[name = string("k_cumdecay_5_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 16, 64, 128]> k_cumdecay_5_cast_fp16 = matmul(transpose_x = k_cumdecay_5_transpose_x_0, transpose_y = k_cumdecay_5_transpose_y_0, x = inv_47_cast_fp16, y = var_1893_cast_fp16)[name = string("k_cumdecay_5_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 128]> q_dec_5_cast_fp16 = mul(x = q_29_cast_fp16, y = var_1892_cast_fp16)[name = string("q_dec_5_cast_fp16")];
+            tensor<fp16, [32, 16, 64]> var_1898_cast_fp16 = exp(x = to_end_5_cast_fp16)[name = string("op_1898_cast_fp16")];
+            tensor<int32, [1]> var_1899_axes_0 = const()[name = string("op_1899_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 16, 64, 1]> var_1899_cast_fp16 = expand_dims(axes = var_1899_axes_0, x = var_1898_cast_fp16)[name = string("op_1899_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 128]> k_dec_5_cast_fp16 = mul(x = k_23_cast_fp16, y = var_1899_cast_fp16)[name = string("k_dec_5_cast_fp16")];
+            tensor<int32, [3]> var_1901_begin_0 = const()[name = string("op_1901_begin_0"), val = tensor<int32, [3]>([0, 0, -1])];
+            tensor<int32, [3]> var_1901_end_0 = const()[name = string("op_1901_end_0"), val = tensor<int32, [3]>([32, 16, 64])];
+            tensor<bool, [3]> var_1901_end_mask_0 = const()[name = string("op_1901_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
+            tensor<fp16, [32, 16, 1]> var_1901_cast_fp16 = slice_by_index(begin = var_1901_begin_0, end = var_1901_end_0, end_mask = var_1901_end_mask_0, x = cum_5_cast_fp16)[name = string("op_1901_cast_fp16")];
+            tensor<fp16, [32, 16, 1]> var_1902_cast_fp16 = exp(x = var_1901_cast_fp16)[name = string("op_1902_cast_fp16")];
+            tensor<int32, [1]> chunk_decay_5_axes_0 = const()[name = string("chunk_decay_5_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 16, 1, 1]> chunk_decay_5_cast_fp16 = expand_dims(axes = chunk_decay_5_axes_0, x = var_1902_cast_fp16)[name = string("chunk_decay_5_cast_fp16")];
+            tensor<int32, [4]> var_1907_begin_0 = const()[name = string("op_1907_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_1907_end_0 = const()[name = string("op_1907_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_1907_end_mask_0 = const()[name = string("op_1907_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1907_squeeze_mask_0 = const()[name = string("op_1907_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1907_cast_fp16 = slice_by_index(begin = var_1907_begin_0, end = var_1907_end_0, end_mask = var_1907_end_mask_0, squeeze_mask = var_1907_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_1907_cast_fp16")];
+            tensor<int32, [4]> var_1909_begin_0 = const()[name = string("op_1909_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_1909_end_0 = const()[name = string("op_1909_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_1909_end_mask_0 = const()[name = string("op_1909_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1909_squeeze_mask_0 = const()[name = string("op_1909_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1909_cast_fp16 = slice_by_index(begin = var_1909_begin_0, end = var_1909_end_0, end_mask = var_1909_end_mask_0, squeeze_mask = var_1909_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_1909_cast_fp16")];
+            bool var_1910_transpose_x_0 = const()[name = string("op_1910_transpose_x_0"), val = bool(false)];
+            bool var_1910_transpose_y_0 = const()[name = string("op_1910_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1910_cast_fp16 = matmul(transpose_x = var_1910_transpose_x_0, transpose_y = var_1910_transpose_y_0, x = var_1909_cast_fp16, y = state_1_to_fp16)[name = string("op_1910_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_65_cast_fp16 = sub(x = var_1907_cast_fp16, y = var_1910_cast_fp16)[name = string("v_new_65_cast_fp16")];
+            tensor<int32, [4]> var_1913_begin_0 = const()[name = string("op_1913_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_1913_end_0 = const()[name = string("op_1913_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_1913_end_mask_0 = const()[name = string("op_1913_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1913_squeeze_mask_0 = const()[name = string("op_1913_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1913_cast_fp16 = slice_by_index(begin = var_1913_begin_0, end = var_1913_end_0, end_mask = var_1913_end_mask_0, squeeze_mask = var_1913_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_1913_cast_fp16")];
+            bool var_1914_transpose_x_0 = const()[name = string("op_1914_transpose_x_0"), val = bool(false)];
+            bool var_1914_transpose_y_0 = const()[name = string("op_1914_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1914_cast_fp16 = matmul(transpose_x = var_1914_transpose_x_0, transpose_y = var_1914_transpose_y_0, x = var_1913_cast_fp16, y = state_1_to_fp16)[name = string("op_1914_cast_fp16")];
+            tensor<int32, [4]> var_1916_begin_0 = const()[name = string("op_1916_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_1916_end_0 = const()[name = string("op_1916_end_0"), val = tensor<int32, [4]>([32, 1, 64, 64])];
+            tensor<bool, [4]> var_1916_end_mask_0 = const()[name = string("op_1916_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1916_squeeze_mask_0 = const()[name = string("op_1916_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1916_cast_fp16 = slice_by_index(begin = var_1916_begin_0, end = var_1916_end_0, end_mask = var_1916_end_mask_0, squeeze_mask = var_1916_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_1916_cast_fp16")];
+            bool var_1917_transpose_x_0 = const()[name = string("op_1917_transpose_x_0"), val = bool(false)];
+            bool var_1917_transpose_y_0 = const()[name = string("op_1917_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1917_cast_fp16 = matmul(transpose_x = var_1917_transpose_x_0, transpose_y = var_1917_transpose_y_0, x = var_1916_cast_fp16, y = v_new_65_cast_fp16)[name = string("op_1917_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1918_cast_fp16 = add(x = var_1914_cast_fp16, y = var_1917_cast_fp16)[name = string("op_1918_cast_fp16")];
+            tensor<int32, [4]> var_1920_begin_0 = const()[name = string("op_1920_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_1920_end_0 = const()[name = string("op_1920_end_0"), val = tensor<int32, [4]>([32, 1, 1, 1])];
+            tensor<bool, [4]> var_1920_end_mask_0 = const()[name = string("op_1920_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1920_squeeze_mask_0 = const()[name = string("op_1920_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1920_cast_fp16 = slice_by_index(begin = var_1920_begin_0, end = var_1920_end_0, end_mask = var_1920_end_mask_0, squeeze_mask = var_1920_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_1920_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1921_cast_fp16 = mul(x = state_1_to_fp16, y = var_1920_cast_fp16)[name = string("op_1921_cast_fp16")];
+            tensor<int32, [4]> var_1923_begin_0 = const()[name = string("op_1923_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_1923_end_0 = const()[name = string("op_1923_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_1923_end_mask_0 = const()[name = string("op_1923_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1923_squeeze_mask_0 = const()[name = string("op_1923_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1923_cast_fp16 = slice_by_index(begin = var_1923_begin_0, end = var_1923_end_0, end_mask = var_1923_end_mask_0, squeeze_mask = var_1923_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_1923_cast_fp16")];
+            bool var_1925_transpose_x_1 = const()[name = string("op_1925_transpose_x_1"), val = bool(true)];
+            bool var_1925_transpose_y_1 = const()[name = string("op_1925_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1925_cast_fp16 = matmul(transpose_x = var_1925_transpose_x_1, transpose_y = var_1925_transpose_y_1, x = var_1923_cast_fp16, y = v_new_65_cast_fp16)[name = string("op_1925_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_67_cast_fp16 = add(x = var_1921_cast_fp16, y = var_1925_cast_fp16)[name = string("state_67_cast_fp16")];
+            tensor<int32, [4]> var_1928_begin_0 = const()[name = string("op_1928_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_1928_end_0 = const()[name = string("op_1928_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_1928_end_mask_0 = const()[name = string("op_1928_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1928_squeeze_mask_0 = const()[name = string("op_1928_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1928_cast_fp16 = slice_by_index(begin = var_1928_begin_0, end = var_1928_end_0, end_mask = var_1928_end_mask_0, squeeze_mask = var_1928_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_1928_cast_fp16")];
+            tensor<int32, [4]> var_1930_begin_0 = const()[name = string("op_1930_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_1930_end_0 = const()[name = string("op_1930_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_1930_end_mask_0 = const()[name = string("op_1930_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1930_squeeze_mask_0 = const()[name = string("op_1930_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1930_cast_fp16 = slice_by_index(begin = var_1930_begin_0, end = var_1930_end_0, end_mask = var_1930_end_mask_0, squeeze_mask = var_1930_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_1930_cast_fp16")];
+            bool var_1931_transpose_x_0 = const()[name = string("op_1931_transpose_x_0"), val = bool(false)];
+            bool var_1931_transpose_y_0 = const()[name = string("op_1931_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1931_cast_fp16 = matmul(transpose_x = var_1931_transpose_x_0, transpose_y = var_1931_transpose_y_0, x = var_1930_cast_fp16, y = state_67_cast_fp16)[name = string("op_1931_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_67_cast_fp16 = sub(x = var_1928_cast_fp16, y = var_1931_cast_fp16)[name = string("v_new_67_cast_fp16")];
+            tensor<int32, [4]> var_1934_begin_0 = const()[name = string("op_1934_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_1934_end_0 = const()[name = string("op_1934_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_1934_end_mask_0 = const()[name = string("op_1934_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1934_squeeze_mask_0 = const()[name = string("op_1934_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1934_cast_fp16 = slice_by_index(begin = var_1934_begin_0, end = var_1934_end_0, end_mask = var_1934_end_mask_0, squeeze_mask = var_1934_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_1934_cast_fp16")];
+            bool var_1935_transpose_x_0 = const()[name = string("op_1935_transpose_x_0"), val = bool(false)];
+            bool var_1935_transpose_y_0 = const()[name = string("op_1935_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1935_cast_fp16 = matmul(transpose_x = var_1935_transpose_x_0, transpose_y = var_1935_transpose_y_0, x = var_1934_cast_fp16, y = state_67_cast_fp16)[name = string("op_1935_cast_fp16")];
+            tensor<int32, [4]> var_1937_begin_0 = const()[name = string("op_1937_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_1937_end_0 = const()[name = string("op_1937_end_0"), val = tensor<int32, [4]>([32, 2, 64, 64])];
+            tensor<bool, [4]> var_1937_end_mask_0 = const()[name = string("op_1937_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1937_squeeze_mask_0 = const()[name = string("op_1937_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1937_cast_fp16 = slice_by_index(begin = var_1937_begin_0, end = var_1937_end_0, end_mask = var_1937_end_mask_0, squeeze_mask = var_1937_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_1937_cast_fp16")];
+            bool var_1938_transpose_x_0 = const()[name = string("op_1938_transpose_x_0"), val = bool(false)];
+            bool var_1938_transpose_y_0 = const()[name = string("op_1938_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1938_cast_fp16 = matmul(transpose_x = var_1938_transpose_x_0, transpose_y = var_1938_transpose_y_0, x = var_1937_cast_fp16, y = v_new_67_cast_fp16)[name = string("op_1938_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1939_cast_fp16 = add(x = var_1935_cast_fp16, y = var_1938_cast_fp16)[name = string("op_1939_cast_fp16")];
+            tensor<int32, [4]> var_1941_begin_0 = const()[name = string("op_1941_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_1941_end_0 = const()[name = string("op_1941_end_0"), val = tensor<int32, [4]>([32, 2, 1, 1])];
+            tensor<bool, [4]> var_1941_end_mask_0 = const()[name = string("op_1941_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1941_squeeze_mask_0 = const()[name = string("op_1941_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1941_cast_fp16 = slice_by_index(begin = var_1941_begin_0, end = var_1941_end_0, end_mask = var_1941_end_mask_0, squeeze_mask = var_1941_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_1941_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1942_cast_fp16 = mul(x = state_67_cast_fp16, y = var_1941_cast_fp16)[name = string("op_1942_cast_fp16")];
+            tensor<int32, [4]> var_1944_begin_0 = const()[name = string("op_1944_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_1944_end_0 = const()[name = string("op_1944_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_1944_end_mask_0 = const()[name = string("op_1944_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1944_squeeze_mask_0 = const()[name = string("op_1944_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1944_cast_fp16 = slice_by_index(begin = var_1944_begin_0, end = var_1944_end_0, end_mask = var_1944_end_mask_0, squeeze_mask = var_1944_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_1944_cast_fp16")];
+            bool var_1946_transpose_x_1 = const()[name = string("op_1946_transpose_x_1"), val = bool(true)];
+            bool var_1946_transpose_y_1 = const()[name = string("op_1946_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1946_cast_fp16 = matmul(transpose_x = var_1946_transpose_x_1, transpose_y = var_1946_transpose_y_1, x = var_1944_cast_fp16, y = v_new_67_cast_fp16)[name = string("op_1946_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_69_cast_fp16 = add(x = var_1942_cast_fp16, y = var_1946_cast_fp16)[name = string("state_69_cast_fp16")];
+            tensor<int32, [4]> var_1949_begin_0 = const()[name = string("op_1949_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_1949_end_0 = const()[name = string("op_1949_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_1949_end_mask_0 = const()[name = string("op_1949_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1949_squeeze_mask_0 = const()[name = string("op_1949_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1949_cast_fp16 = slice_by_index(begin = var_1949_begin_0, end = var_1949_end_0, end_mask = var_1949_end_mask_0, squeeze_mask = var_1949_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_1949_cast_fp16")];
+            tensor<int32, [4]> var_1951_begin_0 = const()[name = string("op_1951_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_1951_end_0 = const()[name = string("op_1951_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_1951_end_mask_0 = const()[name = string("op_1951_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1951_squeeze_mask_0 = const()[name = string("op_1951_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1951_cast_fp16 = slice_by_index(begin = var_1951_begin_0, end = var_1951_end_0, end_mask = var_1951_end_mask_0, squeeze_mask = var_1951_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_1951_cast_fp16")];
+            bool var_1952_transpose_x_0 = const()[name = string("op_1952_transpose_x_0"), val = bool(false)];
+            bool var_1952_transpose_y_0 = const()[name = string("op_1952_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1952_cast_fp16 = matmul(transpose_x = var_1952_transpose_x_0, transpose_y = var_1952_transpose_y_0, x = var_1951_cast_fp16, y = state_69_cast_fp16)[name = string("op_1952_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_69_cast_fp16 = sub(x = var_1949_cast_fp16, y = var_1952_cast_fp16)[name = string("v_new_69_cast_fp16")];
+            tensor<int32, [4]> var_1955_begin_0 = const()[name = string("op_1955_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_1955_end_0 = const()[name = string("op_1955_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_1955_end_mask_0 = const()[name = string("op_1955_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1955_squeeze_mask_0 = const()[name = string("op_1955_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1955_cast_fp16 = slice_by_index(begin = var_1955_begin_0, end = var_1955_end_0, end_mask = var_1955_end_mask_0, squeeze_mask = var_1955_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_1955_cast_fp16")];
+            bool var_1956_transpose_x_0 = const()[name = string("op_1956_transpose_x_0"), val = bool(false)];
+            bool var_1956_transpose_y_0 = const()[name = string("op_1956_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1956_cast_fp16 = matmul(transpose_x = var_1956_transpose_x_0, transpose_y = var_1956_transpose_y_0, x = var_1955_cast_fp16, y = state_69_cast_fp16)[name = string("op_1956_cast_fp16")];
+            tensor<int32, [4]> var_1958_begin_0 = const()[name = string("op_1958_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_1958_end_0 = const()[name = string("op_1958_end_0"), val = tensor<int32, [4]>([32, 3, 64, 64])];
+            tensor<bool, [4]> var_1958_end_mask_0 = const()[name = string("op_1958_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1958_squeeze_mask_0 = const()[name = string("op_1958_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1958_cast_fp16 = slice_by_index(begin = var_1958_begin_0, end = var_1958_end_0, end_mask = var_1958_end_mask_0, squeeze_mask = var_1958_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_1958_cast_fp16")];
+            bool var_1959_transpose_x_0 = const()[name = string("op_1959_transpose_x_0"), val = bool(false)];
+            bool var_1959_transpose_y_0 = const()[name = string("op_1959_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1959_cast_fp16 = matmul(transpose_x = var_1959_transpose_x_0, transpose_y = var_1959_transpose_y_0, x = var_1958_cast_fp16, y = v_new_69_cast_fp16)[name = string("op_1959_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1960_cast_fp16 = add(x = var_1956_cast_fp16, y = var_1959_cast_fp16)[name = string("op_1960_cast_fp16")];
+            tensor<int32, [4]> var_1962_begin_0 = const()[name = string("op_1962_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_1962_end_0 = const()[name = string("op_1962_end_0"), val = tensor<int32, [4]>([32, 3, 1, 1])];
+            tensor<bool, [4]> var_1962_end_mask_0 = const()[name = string("op_1962_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1962_squeeze_mask_0 = const()[name = string("op_1962_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1962_cast_fp16 = slice_by_index(begin = var_1962_begin_0, end = var_1962_end_0, end_mask = var_1962_end_mask_0, squeeze_mask = var_1962_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_1962_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1963_cast_fp16 = mul(x = state_69_cast_fp16, y = var_1962_cast_fp16)[name = string("op_1963_cast_fp16")];
+            tensor<int32, [4]> var_1965_begin_0 = const()[name = string("op_1965_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_1965_end_0 = const()[name = string("op_1965_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_1965_end_mask_0 = const()[name = string("op_1965_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1965_squeeze_mask_0 = const()[name = string("op_1965_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1965_cast_fp16 = slice_by_index(begin = var_1965_begin_0, end = var_1965_end_0, end_mask = var_1965_end_mask_0, squeeze_mask = var_1965_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_1965_cast_fp16")];
+            bool var_1967_transpose_x_1 = const()[name = string("op_1967_transpose_x_1"), val = bool(true)];
+            bool var_1967_transpose_y_1 = const()[name = string("op_1967_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1967_cast_fp16 = matmul(transpose_x = var_1967_transpose_x_1, transpose_y = var_1967_transpose_y_1, x = var_1965_cast_fp16, y = v_new_69_cast_fp16)[name = string("op_1967_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_71_cast_fp16 = add(x = var_1963_cast_fp16, y = var_1967_cast_fp16)[name = string("state_71_cast_fp16")];
+            tensor<int32, [4]> var_1970_begin_0 = const()[name = string("op_1970_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_1970_end_0 = const()[name = string("op_1970_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_1970_end_mask_0 = const()[name = string("op_1970_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1970_squeeze_mask_0 = const()[name = string("op_1970_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1970_cast_fp16 = slice_by_index(begin = var_1970_begin_0, end = var_1970_end_0, end_mask = var_1970_end_mask_0, squeeze_mask = var_1970_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_1970_cast_fp16")];
+            tensor<int32, [4]> var_1972_begin_0 = const()[name = string("op_1972_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_1972_end_0 = const()[name = string("op_1972_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_1972_end_mask_0 = const()[name = string("op_1972_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1972_squeeze_mask_0 = const()[name = string("op_1972_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1972_cast_fp16 = slice_by_index(begin = var_1972_begin_0, end = var_1972_end_0, end_mask = var_1972_end_mask_0, squeeze_mask = var_1972_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_1972_cast_fp16")];
+            bool var_1973_transpose_x_0 = const()[name = string("op_1973_transpose_x_0"), val = bool(false)];
+            bool var_1973_transpose_y_0 = const()[name = string("op_1973_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1973_cast_fp16 = matmul(transpose_x = var_1973_transpose_x_0, transpose_y = var_1973_transpose_y_0, x = var_1972_cast_fp16, y = state_71_cast_fp16)[name = string("op_1973_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_71_cast_fp16 = sub(x = var_1970_cast_fp16, y = var_1973_cast_fp16)[name = string("v_new_71_cast_fp16")];
+            tensor<int32, [4]> var_1976_begin_0 = const()[name = string("op_1976_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_1976_end_0 = const()[name = string("op_1976_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_1976_end_mask_0 = const()[name = string("op_1976_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1976_squeeze_mask_0 = const()[name = string("op_1976_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1976_cast_fp16 = slice_by_index(begin = var_1976_begin_0, end = var_1976_end_0, end_mask = var_1976_end_mask_0, squeeze_mask = var_1976_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_1976_cast_fp16")];
+            bool var_1977_transpose_x_0 = const()[name = string("op_1977_transpose_x_0"), val = bool(false)];
+            bool var_1977_transpose_y_0 = const()[name = string("op_1977_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1977_cast_fp16 = matmul(transpose_x = var_1977_transpose_x_0, transpose_y = var_1977_transpose_y_0, x = var_1976_cast_fp16, y = state_71_cast_fp16)[name = string("op_1977_cast_fp16")];
+            tensor<int32, [4]> var_1979_begin_0 = const()[name = string("op_1979_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_1979_end_0 = const()[name = string("op_1979_end_0"), val = tensor<int32, [4]>([32, 4, 64, 64])];
+            tensor<bool, [4]> var_1979_end_mask_0 = const()[name = string("op_1979_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1979_squeeze_mask_0 = const()[name = string("op_1979_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1979_cast_fp16 = slice_by_index(begin = var_1979_begin_0, end = var_1979_end_0, end_mask = var_1979_end_mask_0, squeeze_mask = var_1979_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_1979_cast_fp16")];
+            bool var_1980_transpose_x_0 = const()[name = string("op_1980_transpose_x_0"), val = bool(false)];
+            bool var_1980_transpose_y_0 = const()[name = string("op_1980_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1980_cast_fp16 = matmul(transpose_x = var_1980_transpose_x_0, transpose_y = var_1980_transpose_y_0, x = var_1979_cast_fp16, y = v_new_71_cast_fp16)[name = string("op_1980_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1981_cast_fp16 = add(x = var_1977_cast_fp16, y = var_1980_cast_fp16)[name = string("op_1981_cast_fp16")];
+            tensor<int32, [4]> var_1983_begin_0 = const()[name = string("op_1983_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_1983_end_0 = const()[name = string("op_1983_end_0"), val = tensor<int32, [4]>([32, 4, 1, 1])];
+            tensor<bool, [4]> var_1983_end_mask_0 = const()[name = string("op_1983_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1983_squeeze_mask_0 = const()[name = string("op_1983_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1983_cast_fp16 = slice_by_index(begin = var_1983_begin_0, end = var_1983_end_0, end_mask = var_1983_end_mask_0, squeeze_mask = var_1983_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_1983_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1984_cast_fp16 = mul(x = state_71_cast_fp16, y = var_1983_cast_fp16)[name = string("op_1984_cast_fp16")];
+            tensor<int32, [4]> var_1986_begin_0 = const()[name = string("op_1986_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_1986_end_0 = const()[name = string("op_1986_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_1986_end_mask_0 = const()[name = string("op_1986_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1986_squeeze_mask_0 = const()[name = string("op_1986_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1986_cast_fp16 = slice_by_index(begin = var_1986_begin_0, end = var_1986_end_0, end_mask = var_1986_end_mask_0, squeeze_mask = var_1986_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_1986_cast_fp16")];
+            bool var_1988_transpose_x_1 = const()[name = string("op_1988_transpose_x_1"), val = bool(true)];
+            bool var_1988_transpose_y_1 = const()[name = string("op_1988_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1988_cast_fp16 = matmul(transpose_x = var_1988_transpose_x_1, transpose_y = var_1988_transpose_y_1, x = var_1986_cast_fp16, y = v_new_71_cast_fp16)[name = string("op_1988_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_73_cast_fp16 = add(x = var_1984_cast_fp16, y = var_1988_cast_fp16)[name = string("state_73_cast_fp16")];
+            tensor<int32, [4]> var_1991_begin_0 = const()[name = string("op_1991_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_1991_end_0 = const()[name = string("op_1991_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_1991_end_mask_0 = const()[name = string("op_1991_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1991_squeeze_mask_0 = const()[name = string("op_1991_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1991_cast_fp16 = slice_by_index(begin = var_1991_begin_0, end = var_1991_end_0, end_mask = var_1991_end_mask_0, squeeze_mask = var_1991_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_1991_cast_fp16")];
+            tensor<int32, [4]> var_1993_begin_0 = const()[name = string("op_1993_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_1993_end_0 = const()[name = string("op_1993_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_1993_end_mask_0 = const()[name = string("op_1993_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1993_squeeze_mask_0 = const()[name = string("op_1993_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1993_cast_fp16 = slice_by_index(begin = var_1993_begin_0, end = var_1993_end_0, end_mask = var_1993_end_mask_0, squeeze_mask = var_1993_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_1993_cast_fp16")];
+            bool var_1994_transpose_x_0 = const()[name = string("op_1994_transpose_x_0"), val = bool(false)];
+            bool var_1994_transpose_y_0 = const()[name = string("op_1994_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1994_cast_fp16 = matmul(transpose_x = var_1994_transpose_x_0, transpose_y = var_1994_transpose_y_0, x = var_1993_cast_fp16, y = state_73_cast_fp16)[name = string("op_1994_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_73_cast_fp16 = sub(x = var_1991_cast_fp16, y = var_1994_cast_fp16)[name = string("v_new_73_cast_fp16")];
+            tensor<int32, [4]> var_1997_begin_0 = const()[name = string("op_1997_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_1997_end_0 = const()[name = string("op_1997_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_1997_end_mask_0 = const()[name = string("op_1997_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1997_squeeze_mask_0 = const()[name = string("op_1997_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1997_cast_fp16 = slice_by_index(begin = var_1997_begin_0, end = var_1997_end_0, end_mask = var_1997_end_mask_0, squeeze_mask = var_1997_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_1997_cast_fp16")];
+            bool var_1998_transpose_x_0 = const()[name = string("op_1998_transpose_x_0"), val = bool(false)];
+            bool var_1998_transpose_y_0 = const()[name = string("op_1998_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1998_cast_fp16 = matmul(transpose_x = var_1998_transpose_x_0, transpose_y = var_1998_transpose_y_0, x = var_1997_cast_fp16, y = state_73_cast_fp16)[name = string("op_1998_cast_fp16")];
+            tensor<int32, [4]> var_2000_begin_0 = const()[name = string("op_2000_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_2000_end_0 = const()[name = string("op_2000_end_0"), val = tensor<int32, [4]>([32, 5, 64, 64])];
+            tensor<bool, [4]> var_2000_end_mask_0 = const()[name = string("op_2000_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2000_squeeze_mask_0 = const()[name = string("op_2000_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2000_cast_fp16 = slice_by_index(begin = var_2000_begin_0, end = var_2000_end_0, end_mask = var_2000_end_mask_0, squeeze_mask = var_2000_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_2000_cast_fp16")];
+            bool var_2001_transpose_x_0 = const()[name = string("op_2001_transpose_x_0"), val = bool(false)];
+            bool var_2001_transpose_y_0 = const()[name = string("op_2001_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2001_cast_fp16 = matmul(transpose_x = var_2001_transpose_x_0, transpose_y = var_2001_transpose_y_0, x = var_2000_cast_fp16, y = v_new_73_cast_fp16)[name = string("op_2001_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2002_cast_fp16 = add(x = var_1998_cast_fp16, y = var_2001_cast_fp16)[name = string("op_2002_cast_fp16")];
+            tensor<int32, [4]> var_2004_begin_0 = const()[name = string("op_2004_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_2004_end_0 = const()[name = string("op_2004_end_0"), val = tensor<int32, [4]>([32, 5, 1, 1])];
+            tensor<bool, [4]> var_2004_end_mask_0 = const()[name = string("op_2004_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2004_squeeze_mask_0 = const()[name = string("op_2004_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2004_cast_fp16 = slice_by_index(begin = var_2004_begin_0, end = var_2004_end_0, end_mask = var_2004_end_mask_0, squeeze_mask = var_2004_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_2004_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2005_cast_fp16 = mul(x = state_73_cast_fp16, y = var_2004_cast_fp16)[name = string("op_2005_cast_fp16")];
+            tensor<int32, [4]> var_2007_begin_0 = const()[name = string("op_2007_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_2007_end_0 = const()[name = string("op_2007_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_2007_end_mask_0 = const()[name = string("op_2007_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2007_squeeze_mask_0 = const()[name = string("op_2007_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2007_cast_fp16 = slice_by_index(begin = var_2007_begin_0, end = var_2007_end_0, end_mask = var_2007_end_mask_0, squeeze_mask = var_2007_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_2007_cast_fp16")];
+            bool var_2009_transpose_x_1 = const()[name = string("op_2009_transpose_x_1"), val = bool(true)];
+            bool var_2009_transpose_y_1 = const()[name = string("op_2009_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2009_cast_fp16 = matmul(transpose_x = var_2009_transpose_x_1, transpose_y = var_2009_transpose_y_1, x = var_2007_cast_fp16, y = v_new_73_cast_fp16)[name = string("op_2009_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_75_cast_fp16 = add(x = var_2005_cast_fp16, y = var_2009_cast_fp16)[name = string("state_75_cast_fp16")];
+            tensor<int32, [4]> var_2012_begin_0 = const()[name = string("op_2012_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_2012_end_0 = const()[name = string("op_2012_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_2012_end_mask_0 = const()[name = string("op_2012_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2012_squeeze_mask_0 = const()[name = string("op_2012_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2012_cast_fp16 = slice_by_index(begin = var_2012_begin_0, end = var_2012_end_0, end_mask = var_2012_end_mask_0, squeeze_mask = var_2012_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_2012_cast_fp16")];
+            tensor<int32, [4]> var_2014_begin_0 = const()[name = string("op_2014_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_2014_end_0 = const()[name = string("op_2014_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_2014_end_mask_0 = const()[name = string("op_2014_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2014_squeeze_mask_0 = const()[name = string("op_2014_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2014_cast_fp16 = slice_by_index(begin = var_2014_begin_0, end = var_2014_end_0, end_mask = var_2014_end_mask_0, squeeze_mask = var_2014_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_2014_cast_fp16")];
+            bool var_2015_transpose_x_0 = const()[name = string("op_2015_transpose_x_0"), val = bool(false)];
+            bool var_2015_transpose_y_0 = const()[name = string("op_2015_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2015_cast_fp16 = matmul(transpose_x = var_2015_transpose_x_0, transpose_y = var_2015_transpose_y_0, x = var_2014_cast_fp16, y = state_75_cast_fp16)[name = string("op_2015_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_75_cast_fp16 = sub(x = var_2012_cast_fp16, y = var_2015_cast_fp16)[name = string("v_new_75_cast_fp16")];
+            tensor<int32, [4]> var_2018_begin_0 = const()[name = string("op_2018_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_2018_end_0 = const()[name = string("op_2018_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_2018_end_mask_0 = const()[name = string("op_2018_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2018_squeeze_mask_0 = const()[name = string("op_2018_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2018_cast_fp16 = slice_by_index(begin = var_2018_begin_0, end = var_2018_end_0, end_mask = var_2018_end_mask_0, squeeze_mask = var_2018_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_2018_cast_fp16")];
+            bool var_2019_transpose_x_0 = const()[name = string("op_2019_transpose_x_0"), val = bool(false)];
+            bool var_2019_transpose_y_0 = const()[name = string("op_2019_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2019_cast_fp16 = matmul(transpose_x = var_2019_transpose_x_0, transpose_y = var_2019_transpose_y_0, x = var_2018_cast_fp16, y = state_75_cast_fp16)[name = string("op_2019_cast_fp16")];
+            tensor<int32, [4]> var_2021_begin_0 = const()[name = string("op_2021_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_2021_end_0 = const()[name = string("op_2021_end_0"), val = tensor<int32, [4]>([32, 6, 64, 64])];
+            tensor<bool, [4]> var_2021_end_mask_0 = const()[name = string("op_2021_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2021_squeeze_mask_0 = const()[name = string("op_2021_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2021_cast_fp16 = slice_by_index(begin = var_2021_begin_0, end = var_2021_end_0, end_mask = var_2021_end_mask_0, squeeze_mask = var_2021_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_2021_cast_fp16")];
+            bool var_2022_transpose_x_0 = const()[name = string("op_2022_transpose_x_0"), val = bool(false)];
+            bool var_2022_transpose_y_0 = const()[name = string("op_2022_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2022_cast_fp16 = matmul(transpose_x = var_2022_transpose_x_0, transpose_y = var_2022_transpose_y_0, x = var_2021_cast_fp16, y = v_new_75_cast_fp16)[name = string("op_2022_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2023_cast_fp16 = add(x = var_2019_cast_fp16, y = var_2022_cast_fp16)[name = string("op_2023_cast_fp16")];
+            tensor<int32, [4]> var_2025_begin_0 = const()[name = string("op_2025_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_2025_end_0 = const()[name = string("op_2025_end_0"), val = tensor<int32, [4]>([32, 6, 1, 1])];
+            tensor<bool, [4]> var_2025_end_mask_0 = const()[name = string("op_2025_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2025_squeeze_mask_0 = const()[name = string("op_2025_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2025_cast_fp16 = slice_by_index(begin = var_2025_begin_0, end = var_2025_end_0, end_mask = var_2025_end_mask_0, squeeze_mask = var_2025_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_2025_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2026_cast_fp16 = mul(x = state_75_cast_fp16, y = var_2025_cast_fp16)[name = string("op_2026_cast_fp16")];
+            tensor<int32, [4]> var_2028_begin_0 = const()[name = string("op_2028_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_2028_end_0 = const()[name = string("op_2028_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_2028_end_mask_0 = const()[name = string("op_2028_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2028_squeeze_mask_0 = const()[name = string("op_2028_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2028_cast_fp16 = slice_by_index(begin = var_2028_begin_0, end = var_2028_end_0, end_mask = var_2028_end_mask_0, squeeze_mask = var_2028_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_2028_cast_fp16")];
+            bool var_2030_transpose_x_1 = const()[name = string("op_2030_transpose_x_1"), val = bool(true)];
+            bool var_2030_transpose_y_1 = const()[name = string("op_2030_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2030_cast_fp16 = matmul(transpose_x = var_2030_transpose_x_1, transpose_y = var_2030_transpose_y_1, x = var_2028_cast_fp16, y = v_new_75_cast_fp16)[name = string("op_2030_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_77_cast_fp16 = add(x = var_2026_cast_fp16, y = var_2030_cast_fp16)[name = string("state_77_cast_fp16")];
+            tensor<int32, [4]> var_2033_begin_0 = const()[name = string("op_2033_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_2033_end_0 = const()[name = string("op_2033_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_2033_end_mask_0 = const()[name = string("op_2033_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2033_squeeze_mask_0 = const()[name = string("op_2033_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2033_cast_fp16 = slice_by_index(begin = var_2033_begin_0, end = var_2033_end_0, end_mask = var_2033_end_mask_0, squeeze_mask = var_2033_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_2033_cast_fp16")];
+            tensor<int32, [4]> var_2035_begin_0 = const()[name = string("op_2035_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_2035_end_0 = const()[name = string("op_2035_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_2035_end_mask_0 = const()[name = string("op_2035_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2035_squeeze_mask_0 = const()[name = string("op_2035_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2035_cast_fp16 = slice_by_index(begin = var_2035_begin_0, end = var_2035_end_0, end_mask = var_2035_end_mask_0, squeeze_mask = var_2035_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_2035_cast_fp16")];
+            bool var_2036_transpose_x_0 = const()[name = string("op_2036_transpose_x_0"), val = bool(false)];
+            bool var_2036_transpose_y_0 = const()[name = string("op_2036_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2036_cast_fp16 = matmul(transpose_x = var_2036_transpose_x_0, transpose_y = var_2036_transpose_y_0, x = var_2035_cast_fp16, y = state_77_cast_fp16)[name = string("op_2036_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_77_cast_fp16 = sub(x = var_2033_cast_fp16, y = var_2036_cast_fp16)[name = string("v_new_77_cast_fp16")];
+            tensor<int32, [4]> var_2039_begin_0 = const()[name = string("op_2039_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_2039_end_0 = const()[name = string("op_2039_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_2039_end_mask_0 = const()[name = string("op_2039_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2039_squeeze_mask_0 = const()[name = string("op_2039_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2039_cast_fp16 = slice_by_index(begin = var_2039_begin_0, end = var_2039_end_0, end_mask = var_2039_end_mask_0, squeeze_mask = var_2039_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_2039_cast_fp16")];
+            bool var_2040_transpose_x_0 = const()[name = string("op_2040_transpose_x_0"), val = bool(false)];
+            bool var_2040_transpose_y_0 = const()[name = string("op_2040_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2040_cast_fp16 = matmul(transpose_x = var_2040_transpose_x_0, transpose_y = var_2040_transpose_y_0, x = var_2039_cast_fp16, y = state_77_cast_fp16)[name = string("op_2040_cast_fp16")];
+            tensor<int32, [4]> var_2042_begin_0 = const()[name = string("op_2042_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_2042_end_0 = const()[name = string("op_2042_end_0"), val = tensor<int32, [4]>([32, 7, 64, 64])];
+            tensor<bool, [4]> var_2042_end_mask_0 = const()[name = string("op_2042_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2042_squeeze_mask_0 = const()[name = string("op_2042_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2042_cast_fp16 = slice_by_index(begin = var_2042_begin_0, end = var_2042_end_0, end_mask = var_2042_end_mask_0, squeeze_mask = var_2042_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_2042_cast_fp16")];
+            bool var_2043_transpose_x_0 = const()[name = string("op_2043_transpose_x_0"), val = bool(false)];
+            bool var_2043_transpose_y_0 = const()[name = string("op_2043_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2043_cast_fp16 = matmul(transpose_x = var_2043_transpose_x_0, transpose_y = var_2043_transpose_y_0, x = var_2042_cast_fp16, y = v_new_77_cast_fp16)[name = string("op_2043_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2044_cast_fp16 = add(x = var_2040_cast_fp16, y = var_2043_cast_fp16)[name = string("op_2044_cast_fp16")];
+            tensor<int32, [4]> var_2046_begin_0 = const()[name = string("op_2046_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_2046_end_0 = const()[name = string("op_2046_end_0"), val = tensor<int32, [4]>([32, 7, 1, 1])];
+            tensor<bool, [4]> var_2046_end_mask_0 = const()[name = string("op_2046_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2046_squeeze_mask_0 = const()[name = string("op_2046_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2046_cast_fp16 = slice_by_index(begin = var_2046_begin_0, end = var_2046_end_0, end_mask = var_2046_end_mask_0, squeeze_mask = var_2046_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_2046_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2047_cast_fp16 = mul(x = state_77_cast_fp16, y = var_2046_cast_fp16)[name = string("op_2047_cast_fp16")];
+            tensor<int32, [4]> var_2049_begin_0 = const()[name = string("op_2049_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_2049_end_0 = const()[name = string("op_2049_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_2049_end_mask_0 = const()[name = string("op_2049_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2049_squeeze_mask_0 = const()[name = string("op_2049_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2049_cast_fp16 = slice_by_index(begin = var_2049_begin_0, end = var_2049_end_0, end_mask = var_2049_end_mask_0, squeeze_mask = var_2049_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_2049_cast_fp16")];
+            bool var_2051_transpose_x_1 = const()[name = string("op_2051_transpose_x_1"), val = bool(true)];
+            bool var_2051_transpose_y_1 = const()[name = string("op_2051_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2051_cast_fp16 = matmul(transpose_x = var_2051_transpose_x_1, transpose_y = var_2051_transpose_y_1, x = var_2049_cast_fp16, y = v_new_77_cast_fp16)[name = string("op_2051_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_79_cast_fp16 = add(x = var_2047_cast_fp16, y = var_2051_cast_fp16)[name = string("state_79_cast_fp16")];
+            tensor<int32, [4]> var_2054_begin_0 = const()[name = string("op_2054_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_2054_end_0 = const()[name = string("op_2054_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_2054_end_mask_0 = const()[name = string("op_2054_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2054_squeeze_mask_0 = const()[name = string("op_2054_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2054_cast_fp16 = slice_by_index(begin = var_2054_begin_0, end = var_2054_end_0, end_mask = var_2054_end_mask_0, squeeze_mask = var_2054_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_2054_cast_fp16")];
+            tensor<int32, [4]> var_2056_begin_0 = const()[name = string("op_2056_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_2056_end_0 = const()[name = string("op_2056_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_2056_end_mask_0 = const()[name = string("op_2056_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2056_squeeze_mask_0 = const()[name = string("op_2056_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2056_cast_fp16 = slice_by_index(begin = var_2056_begin_0, end = var_2056_end_0, end_mask = var_2056_end_mask_0, squeeze_mask = var_2056_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_2056_cast_fp16")];
+            bool var_2057_transpose_x_0 = const()[name = string("op_2057_transpose_x_0"), val = bool(false)];
+            bool var_2057_transpose_y_0 = const()[name = string("op_2057_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2057_cast_fp16 = matmul(transpose_x = var_2057_transpose_x_0, transpose_y = var_2057_transpose_y_0, x = var_2056_cast_fp16, y = state_79_cast_fp16)[name = string("op_2057_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_79_cast_fp16 = sub(x = var_2054_cast_fp16, y = var_2057_cast_fp16)[name = string("v_new_79_cast_fp16")];
+            tensor<int32, [4]> var_2060_begin_0 = const()[name = string("op_2060_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_2060_end_0 = const()[name = string("op_2060_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_2060_end_mask_0 = const()[name = string("op_2060_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2060_squeeze_mask_0 = const()[name = string("op_2060_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2060_cast_fp16 = slice_by_index(begin = var_2060_begin_0, end = var_2060_end_0, end_mask = var_2060_end_mask_0, squeeze_mask = var_2060_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_2060_cast_fp16")];
+            bool var_2061_transpose_x_0 = const()[name = string("op_2061_transpose_x_0"), val = bool(false)];
+            bool var_2061_transpose_y_0 = const()[name = string("op_2061_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2061_cast_fp16 = matmul(transpose_x = var_2061_transpose_x_0, transpose_y = var_2061_transpose_y_0, x = var_2060_cast_fp16, y = state_79_cast_fp16)[name = string("op_2061_cast_fp16")];
+            tensor<int32, [4]> var_2063_begin_0 = const()[name = string("op_2063_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_2063_end_0 = const()[name = string("op_2063_end_0"), val = tensor<int32, [4]>([32, 8, 64, 64])];
+            tensor<bool, [4]> var_2063_end_mask_0 = const()[name = string("op_2063_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2063_squeeze_mask_0 = const()[name = string("op_2063_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2063_cast_fp16 = slice_by_index(begin = var_2063_begin_0, end = var_2063_end_0, end_mask = var_2063_end_mask_0, squeeze_mask = var_2063_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_2063_cast_fp16")];
+            bool var_2064_transpose_x_0 = const()[name = string("op_2064_transpose_x_0"), val = bool(false)];
+            bool var_2064_transpose_y_0 = const()[name = string("op_2064_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2064_cast_fp16 = matmul(transpose_x = var_2064_transpose_x_0, transpose_y = var_2064_transpose_y_0, x = var_2063_cast_fp16, y = v_new_79_cast_fp16)[name = string("op_2064_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2065_cast_fp16 = add(x = var_2061_cast_fp16, y = var_2064_cast_fp16)[name = string("op_2065_cast_fp16")];
+            tensor<int32, [4]> var_2067_begin_0 = const()[name = string("op_2067_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_2067_end_0 = const()[name = string("op_2067_end_0"), val = tensor<int32, [4]>([32, 8, 1, 1])];
+            tensor<bool, [4]> var_2067_end_mask_0 = const()[name = string("op_2067_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2067_squeeze_mask_0 = const()[name = string("op_2067_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2067_cast_fp16 = slice_by_index(begin = var_2067_begin_0, end = var_2067_end_0, end_mask = var_2067_end_mask_0, squeeze_mask = var_2067_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_2067_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2068_cast_fp16 = mul(x = state_79_cast_fp16, y = var_2067_cast_fp16)[name = string("op_2068_cast_fp16")];
+            tensor<int32, [4]> var_2070_begin_0 = const()[name = string("op_2070_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_2070_end_0 = const()[name = string("op_2070_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_2070_end_mask_0 = const()[name = string("op_2070_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2070_squeeze_mask_0 = const()[name = string("op_2070_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2070_cast_fp16 = slice_by_index(begin = var_2070_begin_0, end = var_2070_end_0, end_mask = var_2070_end_mask_0, squeeze_mask = var_2070_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_2070_cast_fp16")];
+            bool var_2072_transpose_x_1 = const()[name = string("op_2072_transpose_x_1"), val = bool(true)];
+            bool var_2072_transpose_y_1 = const()[name = string("op_2072_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2072_cast_fp16 = matmul(transpose_x = var_2072_transpose_x_1, transpose_y = var_2072_transpose_y_1, x = var_2070_cast_fp16, y = v_new_79_cast_fp16)[name = string("op_2072_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_81_cast_fp16 = add(x = var_2068_cast_fp16, y = var_2072_cast_fp16)[name = string("state_81_cast_fp16")];
+            tensor<int32, [4]> var_2075_begin_0 = const()[name = string("op_2075_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_2075_end_0 = const()[name = string("op_2075_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_2075_end_mask_0 = const()[name = string("op_2075_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2075_squeeze_mask_0 = const()[name = string("op_2075_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2075_cast_fp16 = slice_by_index(begin = var_2075_begin_0, end = var_2075_end_0, end_mask = var_2075_end_mask_0, squeeze_mask = var_2075_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_2075_cast_fp16")];
+            tensor<int32, [4]> var_2077_begin_0 = const()[name = string("op_2077_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_2077_end_0 = const()[name = string("op_2077_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_2077_end_mask_0 = const()[name = string("op_2077_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2077_squeeze_mask_0 = const()[name = string("op_2077_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2077_cast_fp16 = slice_by_index(begin = var_2077_begin_0, end = var_2077_end_0, end_mask = var_2077_end_mask_0, squeeze_mask = var_2077_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_2077_cast_fp16")];
+            bool var_2078_transpose_x_0 = const()[name = string("op_2078_transpose_x_0"), val = bool(false)];
+            bool var_2078_transpose_y_0 = const()[name = string("op_2078_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2078_cast_fp16 = matmul(transpose_x = var_2078_transpose_x_0, transpose_y = var_2078_transpose_y_0, x = var_2077_cast_fp16, y = state_81_cast_fp16)[name = string("op_2078_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_81_cast_fp16 = sub(x = var_2075_cast_fp16, y = var_2078_cast_fp16)[name = string("v_new_81_cast_fp16")];
+            tensor<int32, [4]> var_2081_begin_0 = const()[name = string("op_2081_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_2081_end_0 = const()[name = string("op_2081_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_2081_end_mask_0 = const()[name = string("op_2081_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2081_squeeze_mask_0 = const()[name = string("op_2081_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2081_cast_fp16 = slice_by_index(begin = var_2081_begin_0, end = var_2081_end_0, end_mask = var_2081_end_mask_0, squeeze_mask = var_2081_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_2081_cast_fp16")];
+            bool var_2082_transpose_x_0 = const()[name = string("op_2082_transpose_x_0"), val = bool(false)];
+            bool var_2082_transpose_y_0 = const()[name = string("op_2082_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2082_cast_fp16 = matmul(transpose_x = var_2082_transpose_x_0, transpose_y = var_2082_transpose_y_0, x = var_2081_cast_fp16, y = state_81_cast_fp16)[name = string("op_2082_cast_fp16")];
+            tensor<int32, [4]> var_2084_begin_0 = const()[name = string("op_2084_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_2084_end_0 = const()[name = string("op_2084_end_0"), val = tensor<int32, [4]>([32, 9, 64, 64])];
+            tensor<bool, [4]> var_2084_end_mask_0 = const()[name = string("op_2084_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2084_squeeze_mask_0 = const()[name = string("op_2084_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2084_cast_fp16 = slice_by_index(begin = var_2084_begin_0, end = var_2084_end_0, end_mask = var_2084_end_mask_0, squeeze_mask = var_2084_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_2084_cast_fp16")];
+            bool var_2085_transpose_x_0 = const()[name = string("op_2085_transpose_x_0"), val = bool(false)];
+            bool var_2085_transpose_y_0 = const()[name = string("op_2085_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2085_cast_fp16 = matmul(transpose_x = var_2085_transpose_x_0, transpose_y = var_2085_transpose_y_0, x = var_2084_cast_fp16, y = v_new_81_cast_fp16)[name = string("op_2085_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2086_cast_fp16 = add(x = var_2082_cast_fp16, y = var_2085_cast_fp16)[name = string("op_2086_cast_fp16")];
+            tensor<int32, [4]> var_2088_begin_0 = const()[name = string("op_2088_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_2088_end_0 = const()[name = string("op_2088_end_0"), val = tensor<int32, [4]>([32, 9, 1, 1])];
+            tensor<bool, [4]> var_2088_end_mask_0 = const()[name = string("op_2088_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2088_squeeze_mask_0 = const()[name = string("op_2088_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2088_cast_fp16 = slice_by_index(begin = var_2088_begin_0, end = var_2088_end_0, end_mask = var_2088_end_mask_0, squeeze_mask = var_2088_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_2088_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2089_cast_fp16 = mul(x = state_81_cast_fp16, y = var_2088_cast_fp16)[name = string("op_2089_cast_fp16")];
+            tensor<int32, [4]> var_2091_begin_0 = const()[name = string("op_2091_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_2091_end_0 = const()[name = string("op_2091_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_2091_end_mask_0 = const()[name = string("op_2091_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2091_squeeze_mask_0 = const()[name = string("op_2091_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2091_cast_fp16 = slice_by_index(begin = var_2091_begin_0, end = var_2091_end_0, end_mask = var_2091_end_mask_0, squeeze_mask = var_2091_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_2091_cast_fp16")];
+            bool var_2093_transpose_x_1 = const()[name = string("op_2093_transpose_x_1"), val = bool(true)];
+            bool var_2093_transpose_y_1 = const()[name = string("op_2093_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2093_cast_fp16 = matmul(transpose_x = var_2093_transpose_x_1, transpose_y = var_2093_transpose_y_1, x = var_2091_cast_fp16, y = v_new_81_cast_fp16)[name = string("op_2093_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_83_cast_fp16 = add(x = var_2089_cast_fp16, y = var_2093_cast_fp16)[name = string("state_83_cast_fp16")];
+            tensor<int32, [4]> var_2096_begin_0 = const()[name = string("op_2096_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_2096_end_0 = const()[name = string("op_2096_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_2096_end_mask_0 = const()[name = string("op_2096_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2096_squeeze_mask_0 = const()[name = string("op_2096_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2096_cast_fp16 = slice_by_index(begin = var_2096_begin_0, end = var_2096_end_0, end_mask = var_2096_end_mask_0, squeeze_mask = var_2096_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_2096_cast_fp16")];
+            tensor<int32, [4]> var_2098_begin_0 = const()[name = string("op_2098_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_2098_end_0 = const()[name = string("op_2098_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_2098_end_mask_0 = const()[name = string("op_2098_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2098_squeeze_mask_0 = const()[name = string("op_2098_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2098_cast_fp16 = slice_by_index(begin = var_2098_begin_0, end = var_2098_end_0, end_mask = var_2098_end_mask_0, squeeze_mask = var_2098_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_2098_cast_fp16")];
+            bool var_2099_transpose_x_0 = const()[name = string("op_2099_transpose_x_0"), val = bool(false)];
+            bool var_2099_transpose_y_0 = const()[name = string("op_2099_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2099_cast_fp16 = matmul(transpose_x = var_2099_transpose_x_0, transpose_y = var_2099_transpose_y_0, x = var_2098_cast_fp16, y = state_83_cast_fp16)[name = string("op_2099_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_83_cast_fp16 = sub(x = var_2096_cast_fp16, y = var_2099_cast_fp16)[name = string("v_new_83_cast_fp16")];
+            tensor<int32, [4]> var_2102_begin_0 = const()[name = string("op_2102_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_2102_end_0 = const()[name = string("op_2102_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_2102_end_mask_0 = const()[name = string("op_2102_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2102_squeeze_mask_0 = const()[name = string("op_2102_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2102_cast_fp16 = slice_by_index(begin = var_2102_begin_0, end = var_2102_end_0, end_mask = var_2102_end_mask_0, squeeze_mask = var_2102_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_2102_cast_fp16")];
+            bool var_2103_transpose_x_0 = const()[name = string("op_2103_transpose_x_0"), val = bool(false)];
+            bool var_2103_transpose_y_0 = const()[name = string("op_2103_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2103_cast_fp16 = matmul(transpose_x = var_2103_transpose_x_0, transpose_y = var_2103_transpose_y_0, x = var_2102_cast_fp16, y = state_83_cast_fp16)[name = string("op_2103_cast_fp16")];
+            tensor<int32, [4]> var_2105_begin_0 = const()[name = string("op_2105_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_2105_end_0 = const()[name = string("op_2105_end_0"), val = tensor<int32, [4]>([32, 10, 64, 64])];
+            tensor<bool, [4]> var_2105_end_mask_0 = const()[name = string("op_2105_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2105_squeeze_mask_0 = const()[name = string("op_2105_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2105_cast_fp16 = slice_by_index(begin = var_2105_begin_0, end = var_2105_end_0, end_mask = var_2105_end_mask_0, squeeze_mask = var_2105_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_2105_cast_fp16")];
+            bool var_2106_transpose_x_0 = const()[name = string("op_2106_transpose_x_0"), val = bool(false)];
+            bool var_2106_transpose_y_0 = const()[name = string("op_2106_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2106_cast_fp16 = matmul(transpose_x = var_2106_transpose_x_0, transpose_y = var_2106_transpose_y_0, x = var_2105_cast_fp16, y = v_new_83_cast_fp16)[name = string("op_2106_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2107_cast_fp16 = add(x = var_2103_cast_fp16, y = var_2106_cast_fp16)[name = string("op_2107_cast_fp16")];
+            tensor<int32, [4]> var_2109_begin_0 = const()[name = string("op_2109_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_2109_end_0 = const()[name = string("op_2109_end_0"), val = tensor<int32, [4]>([32, 10, 1, 1])];
+            tensor<bool, [4]> var_2109_end_mask_0 = const()[name = string("op_2109_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2109_squeeze_mask_0 = const()[name = string("op_2109_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2109_cast_fp16 = slice_by_index(begin = var_2109_begin_0, end = var_2109_end_0, end_mask = var_2109_end_mask_0, squeeze_mask = var_2109_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_2109_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2110_cast_fp16 = mul(x = state_83_cast_fp16, y = var_2109_cast_fp16)[name = string("op_2110_cast_fp16")];
+            tensor<int32, [4]> var_2112_begin_0 = const()[name = string("op_2112_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_2112_end_0 = const()[name = string("op_2112_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_2112_end_mask_0 = const()[name = string("op_2112_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2112_squeeze_mask_0 = const()[name = string("op_2112_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2112_cast_fp16 = slice_by_index(begin = var_2112_begin_0, end = var_2112_end_0, end_mask = var_2112_end_mask_0, squeeze_mask = var_2112_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_2112_cast_fp16")];
+            bool var_2114_transpose_x_1 = const()[name = string("op_2114_transpose_x_1"), val = bool(true)];
+            bool var_2114_transpose_y_1 = const()[name = string("op_2114_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2114_cast_fp16 = matmul(transpose_x = var_2114_transpose_x_1, transpose_y = var_2114_transpose_y_1, x = var_2112_cast_fp16, y = v_new_83_cast_fp16)[name = string("op_2114_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_85_cast_fp16 = add(x = var_2110_cast_fp16, y = var_2114_cast_fp16)[name = string("state_85_cast_fp16")];
+            tensor<int32, [4]> var_2117_begin_0 = const()[name = string("op_2117_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_2117_end_0 = const()[name = string("op_2117_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_2117_end_mask_0 = const()[name = string("op_2117_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2117_squeeze_mask_0 = const()[name = string("op_2117_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2117_cast_fp16 = slice_by_index(begin = var_2117_begin_0, end = var_2117_end_0, end_mask = var_2117_end_mask_0, squeeze_mask = var_2117_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_2117_cast_fp16")];
+            tensor<int32, [4]> var_2119_begin_0 = const()[name = string("op_2119_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_2119_end_0 = const()[name = string("op_2119_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_2119_end_mask_0 = const()[name = string("op_2119_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2119_squeeze_mask_0 = const()[name = string("op_2119_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2119_cast_fp16 = slice_by_index(begin = var_2119_begin_0, end = var_2119_end_0, end_mask = var_2119_end_mask_0, squeeze_mask = var_2119_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_2119_cast_fp16")];
+            bool var_2120_transpose_x_0 = const()[name = string("op_2120_transpose_x_0"), val = bool(false)];
+            bool var_2120_transpose_y_0 = const()[name = string("op_2120_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2120_cast_fp16 = matmul(transpose_x = var_2120_transpose_x_0, transpose_y = var_2120_transpose_y_0, x = var_2119_cast_fp16, y = state_85_cast_fp16)[name = string("op_2120_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_85_cast_fp16 = sub(x = var_2117_cast_fp16, y = var_2120_cast_fp16)[name = string("v_new_85_cast_fp16")];
+            tensor<int32, [4]> var_2123_begin_0 = const()[name = string("op_2123_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_2123_end_0 = const()[name = string("op_2123_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_2123_end_mask_0 = const()[name = string("op_2123_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2123_squeeze_mask_0 = const()[name = string("op_2123_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2123_cast_fp16 = slice_by_index(begin = var_2123_begin_0, end = var_2123_end_0, end_mask = var_2123_end_mask_0, squeeze_mask = var_2123_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_2123_cast_fp16")];
+            bool var_2124_transpose_x_0 = const()[name = string("op_2124_transpose_x_0"), val = bool(false)];
+            bool var_2124_transpose_y_0 = const()[name = string("op_2124_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2124_cast_fp16 = matmul(transpose_x = var_2124_transpose_x_0, transpose_y = var_2124_transpose_y_0, x = var_2123_cast_fp16, y = state_85_cast_fp16)[name = string("op_2124_cast_fp16")];
+            tensor<int32, [4]> var_2126_begin_0 = const()[name = string("op_2126_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_2126_end_0 = const()[name = string("op_2126_end_0"), val = tensor<int32, [4]>([32, 11, 64, 64])];
+            tensor<bool, [4]> var_2126_end_mask_0 = const()[name = string("op_2126_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2126_squeeze_mask_0 = const()[name = string("op_2126_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2126_cast_fp16 = slice_by_index(begin = var_2126_begin_0, end = var_2126_end_0, end_mask = var_2126_end_mask_0, squeeze_mask = var_2126_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_2126_cast_fp16")];
+            bool var_2127_transpose_x_0 = const()[name = string("op_2127_transpose_x_0"), val = bool(false)];
+            bool var_2127_transpose_y_0 = const()[name = string("op_2127_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2127_cast_fp16 = matmul(transpose_x = var_2127_transpose_x_0, transpose_y = var_2127_transpose_y_0, x = var_2126_cast_fp16, y = v_new_85_cast_fp16)[name = string("op_2127_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2128_cast_fp16 = add(x = var_2124_cast_fp16, y = var_2127_cast_fp16)[name = string("op_2128_cast_fp16")];
+            tensor<int32, [4]> var_2130_begin_0 = const()[name = string("op_2130_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_2130_end_0 = const()[name = string("op_2130_end_0"), val = tensor<int32, [4]>([32, 11, 1, 1])];
+            tensor<bool, [4]> var_2130_end_mask_0 = const()[name = string("op_2130_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2130_squeeze_mask_0 = const()[name = string("op_2130_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2130_cast_fp16 = slice_by_index(begin = var_2130_begin_0, end = var_2130_end_0, end_mask = var_2130_end_mask_0, squeeze_mask = var_2130_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_2130_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2131_cast_fp16 = mul(x = state_85_cast_fp16, y = var_2130_cast_fp16)[name = string("op_2131_cast_fp16")];
+            tensor<int32, [4]> var_2133_begin_0 = const()[name = string("op_2133_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_2133_end_0 = const()[name = string("op_2133_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_2133_end_mask_0 = const()[name = string("op_2133_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2133_squeeze_mask_0 = const()[name = string("op_2133_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2133_cast_fp16 = slice_by_index(begin = var_2133_begin_0, end = var_2133_end_0, end_mask = var_2133_end_mask_0, squeeze_mask = var_2133_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_2133_cast_fp16")];
+            bool var_2135_transpose_x_1 = const()[name = string("op_2135_transpose_x_1"), val = bool(true)];
+            bool var_2135_transpose_y_1 = const()[name = string("op_2135_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2135_cast_fp16 = matmul(transpose_x = var_2135_transpose_x_1, transpose_y = var_2135_transpose_y_1, x = var_2133_cast_fp16, y = v_new_85_cast_fp16)[name = string("op_2135_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_87_cast_fp16 = add(x = var_2131_cast_fp16, y = var_2135_cast_fp16)[name = string("state_87_cast_fp16")];
+            tensor<int32, [4]> var_2138_begin_0 = const()[name = string("op_2138_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_2138_end_0 = const()[name = string("op_2138_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_2138_end_mask_0 = const()[name = string("op_2138_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2138_squeeze_mask_0 = const()[name = string("op_2138_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2138_cast_fp16 = slice_by_index(begin = var_2138_begin_0, end = var_2138_end_0, end_mask = var_2138_end_mask_0, squeeze_mask = var_2138_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_2138_cast_fp16")];
+            tensor<int32, [4]> var_2140_begin_0 = const()[name = string("op_2140_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_2140_end_0 = const()[name = string("op_2140_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_2140_end_mask_0 = const()[name = string("op_2140_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2140_squeeze_mask_0 = const()[name = string("op_2140_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2140_cast_fp16 = slice_by_index(begin = var_2140_begin_0, end = var_2140_end_0, end_mask = var_2140_end_mask_0, squeeze_mask = var_2140_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_2140_cast_fp16")];
+            bool var_2141_transpose_x_0 = const()[name = string("op_2141_transpose_x_0"), val = bool(false)];
+            bool var_2141_transpose_y_0 = const()[name = string("op_2141_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2141_cast_fp16 = matmul(transpose_x = var_2141_transpose_x_0, transpose_y = var_2141_transpose_y_0, x = var_2140_cast_fp16, y = state_87_cast_fp16)[name = string("op_2141_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_87_cast_fp16 = sub(x = var_2138_cast_fp16, y = var_2141_cast_fp16)[name = string("v_new_87_cast_fp16")];
+            tensor<int32, [4]> var_2144_begin_0 = const()[name = string("op_2144_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_2144_end_0 = const()[name = string("op_2144_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_2144_end_mask_0 = const()[name = string("op_2144_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2144_squeeze_mask_0 = const()[name = string("op_2144_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2144_cast_fp16 = slice_by_index(begin = var_2144_begin_0, end = var_2144_end_0, end_mask = var_2144_end_mask_0, squeeze_mask = var_2144_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_2144_cast_fp16")];
+            bool var_2145_transpose_x_0 = const()[name = string("op_2145_transpose_x_0"), val = bool(false)];
+            bool var_2145_transpose_y_0 = const()[name = string("op_2145_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2145_cast_fp16 = matmul(transpose_x = var_2145_transpose_x_0, transpose_y = var_2145_transpose_y_0, x = var_2144_cast_fp16, y = state_87_cast_fp16)[name = string("op_2145_cast_fp16")];
+            tensor<int32, [4]> var_2147_begin_0 = const()[name = string("op_2147_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_2147_end_0 = const()[name = string("op_2147_end_0"), val = tensor<int32, [4]>([32, 12, 64, 64])];
+            tensor<bool, [4]> var_2147_end_mask_0 = const()[name = string("op_2147_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2147_squeeze_mask_0 = const()[name = string("op_2147_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2147_cast_fp16 = slice_by_index(begin = var_2147_begin_0, end = var_2147_end_0, end_mask = var_2147_end_mask_0, squeeze_mask = var_2147_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_2147_cast_fp16")];
+            bool var_2148_transpose_x_0 = const()[name = string("op_2148_transpose_x_0"), val = bool(false)];
+            bool var_2148_transpose_y_0 = const()[name = string("op_2148_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2148_cast_fp16 = matmul(transpose_x = var_2148_transpose_x_0, transpose_y = var_2148_transpose_y_0, x = var_2147_cast_fp16, y = v_new_87_cast_fp16)[name = string("op_2148_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2149_cast_fp16 = add(x = var_2145_cast_fp16, y = var_2148_cast_fp16)[name = string("op_2149_cast_fp16")];
+            tensor<int32, [4]> var_2151_begin_0 = const()[name = string("op_2151_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_2151_end_0 = const()[name = string("op_2151_end_0"), val = tensor<int32, [4]>([32, 12, 1, 1])];
+            tensor<bool, [4]> var_2151_end_mask_0 = const()[name = string("op_2151_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2151_squeeze_mask_0 = const()[name = string("op_2151_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2151_cast_fp16 = slice_by_index(begin = var_2151_begin_0, end = var_2151_end_0, end_mask = var_2151_end_mask_0, squeeze_mask = var_2151_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_2151_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2152_cast_fp16 = mul(x = state_87_cast_fp16, y = var_2151_cast_fp16)[name = string("op_2152_cast_fp16")];
+            tensor<int32, [4]> var_2154_begin_0 = const()[name = string("op_2154_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_2154_end_0 = const()[name = string("op_2154_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_2154_end_mask_0 = const()[name = string("op_2154_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2154_squeeze_mask_0 = const()[name = string("op_2154_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2154_cast_fp16 = slice_by_index(begin = var_2154_begin_0, end = var_2154_end_0, end_mask = var_2154_end_mask_0, squeeze_mask = var_2154_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_2154_cast_fp16")];
+            bool var_2156_transpose_x_1 = const()[name = string("op_2156_transpose_x_1"), val = bool(true)];
+            bool var_2156_transpose_y_1 = const()[name = string("op_2156_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2156_cast_fp16 = matmul(transpose_x = var_2156_transpose_x_1, transpose_y = var_2156_transpose_y_1, x = var_2154_cast_fp16, y = v_new_87_cast_fp16)[name = string("op_2156_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_89_cast_fp16 = add(x = var_2152_cast_fp16, y = var_2156_cast_fp16)[name = string("state_89_cast_fp16")];
+            tensor<int32, [4]> var_2159_begin_0 = const()[name = string("op_2159_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_2159_end_0 = const()[name = string("op_2159_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_2159_end_mask_0 = const()[name = string("op_2159_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2159_squeeze_mask_0 = const()[name = string("op_2159_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2159_cast_fp16 = slice_by_index(begin = var_2159_begin_0, end = var_2159_end_0, end_mask = var_2159_end_mask_0, squeeze_mask = var_2159_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_2159_cast_fp16")];
+            tensor<int32, [4]> var_2161_begin_0 = const()[name = string("op_2161_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_2161_end_0 = const()[name = string("op_2161_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_2161_end_mask_0 = const()[name = string("op_2161_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2161_squeeze_mask_0 = const()[name = string("op_2161_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2161_cast_fp16 = slice_by_index(begin = var_2161_begin_0, end = var_2161_end_0, end_mask = var_2161_end_mask_0, squeeze_mask = var_2161_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_2161_cast_fp16")];
+            bool var_2162_transpose_x_0 = const()[name = string("op_2162_transpose_x_0"), val = bool(false)];
+            bool var_2162_transpose_y_0 = const()[name = string("op_2162_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2162_cast_fp16 = matmul(transpose_x = var_2162_transpose_x_0, transpose_y = var_2162_transpose_y_0, x = var_2161_cast_fp16, y = state_89_cast_fp16)[name = string("op_2162_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_89_cast_fp16 = sub(x = var_2159_cast_fp16, y = var_2162_cast_fp16)[name = string("v_new_89_cast_fp16")];
+            tensor<int32, [4]> var_2165_begin_0 = const()[name = string("op_2165_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_2165_end_0 = const()[name = string("op_2165_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_2165_end_mask_0 = const()[name = string("op_2165_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2165_squeeze_mask_0 = const()[name = string("op_2165_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2165_cast_fp16 = slice_by_index(begin = var_2165_begin_0, end = var_2165_end_0, end_mask = var_2165_end_mask_0, squeeze_mask = var_2165_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_2165_cast_fp16")];
+            bool var_2166_transpose_x_0 = const()[name = string("op_2166_transpose_x_0"), val = bool(false)];
+            bool var_2166_transpose_y_0 = const()[name = string("op_2166_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2166_cast_fp16 = matmul(transpose_x = var_2166_transpose_x_0, transpose_y = var_2166_transpose_y_0, x = var_2165_cast_fp16, y = state_89_cast_fp16)[name = string("op_2166_cast_fp16")];
+            tensor<int32, [4]> var_2168_begin_0 = const()[name = string("op_2168_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_2168_end_0 = const()[name = string("op_2168_end_0"), val = tensor<int32, [4]>([32, 13, 64, 64])];
+            tensor<bool, [4]> var_2168_end_mask_0 = const()[name = string("op_2168_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2168_squeeze_mask_0 = const()[name = string("op_2168_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2168_cast_fp16 = slice_by_index(begin = var_2168_begin_0, end = var_2168_end_0, end_mask = var_2168_end_mask_0, squeeze_mask = var_2168_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_2168_cast_fp16")];
+            bool var_2169_transpose_x_0 = const()[name = string("op_2169_transpose_x_0"), val = bool(false)];
+            bool var_2169_transpose_y_0 = const()[name = string("op_2169_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2169_cast_fp16 = matmul(transpose_x = var_2169_transpose_x_0, transpose_y = var_2169_transpose_y_0, x = var_2168_cast_fp16, y = v_new_89_cast_fp16)[name = string("op_2169_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2170_cast_fp16 = add(x = var_2166_cast_fp16, y = var_2169_cast_fp16)[name = string("op_2170_cast_fp16")];
+            tensor<int32, [4]> var_2172_begin_0 = const()[name = string("op_2172_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_2172_end_0 = const()[name = string("op_2172_end_0"), val = tensor<int32, [4]>([32, 13, 1, 1])];
+            tensor<bool, [4]> var_2172_end_mask_0 = const()[name = string("op_2172_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2172_squeeze_mask_0 = const()[name = string("op_2172_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2172_cast_fp16 = slice_by_index(begin = var_2172_begin_0, end = var_2172_end_0, end_mask = var_2172_end_mask_0, squeeze_mask = var_2172_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_2172_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2173_cast_fp16 = mul(x = state_89_cast_fp16, y = var_2172_cast_fp16)[name = string("op_2173_cast_fp16")];
+            tensor<int32, [4]> var_2175_begin_0 = const()[name = string("op_2175_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_2175_end_0 = const()[name = string("op_2175_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_2175_end_mask_0 = const()[name = string("op_2175_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2175_squeeze_mask_0 = const()[name = string("op_2175_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2175_cast_fp16 = slice_by_index(begin = var_2175_begin_0, end = var_2175_end_0, end_mask = var_2175_end_mask_0, squeeze_mask = var_2175_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_2175_cast_fp16")];
+            bool var_2177_transpose_x_1 = const()[name = string("op_2177_transpose_x_1"), val = bool(true)];
+            bool var_2177_transpose_y_1 = const()[name = string("op_2177_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2177_cast_fp16 = matmul(transpose_x = var_2177_transpose_x_1, transpose_y = var_2177_transpose_y_1, x = var_2175_cast_fp16, y = v_new_89_cast_fp16)[name = string("op_2177_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_91_cast_fp16 = add(x = var_2173_cast_fp16, y = var_2177_cast_fp16)[name = string("state_91_cast_fp16")];
+            tensor<int32, [4]> var_2180_begin_0 = const()[name = string("op_2180_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_2180_end_0 = const()[name = string("op_2180_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_2180_end_mask_0 = const()[name = string("op_2180_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2180_squeeze_mask_0 = const()[name = string("op_2180_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2180_cast_fp16 = slice_by_index(begin = var_2180_begin_0, end = var_2180_end_0, end_mask = var_2180_end_mask_0, squeeze_mask = var_2180_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_2180_cast_fp16")];
+            tensor<int32, [4]> var_2182_begin_0 = const()[name = string("op_2182_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_2182_end_0 = const()[name = string("op_2182_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_2182_end_mask_0 = const()[name = string("op_2182_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2182_squeeze_mask_0 = const()[name = string("op_2182_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2182_cast_fp16 = slice_by_index(begin = var_2182_begin_0, end = var_2182_end_0, end_mask = var_2182_end_mask_0, squeeze_mask = var_2182_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_2182_cast_fp16")];
+            bool var_2183_transpose_x_0 = const()[name = string("op_2183_transpose_x_0"), val = bool(false)];
+            bool var_2183_transpose_y_0 = const()[name = string("op_2183_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2183_cast_fp16 = matmul(transpose_x = var_2183_transpose_x_0, transpose_y = var_2183_transpose_y_0, x = var_2182_cast_fp16, y = state_91_cast_fp16)[name = string("op_2183_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_91_cast_fp16 = sub(x = var_2180_cast_fp16, y = var_2183_cast_fp16)[name = string("v_new_91_cast_fp16")];
+            tensor<int32, [4]> var_2186_begin_0 = const()[name = string("op_2186_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_2186_end_0 = const()[name = string("op_2186_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_2186_end_mask_0 = const()[name = string("op_2186_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2186_squeeze_mask_0 = const()[name = string("op_2186_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2186_cast_fp16 = slice_by_index(begin = var_2186_begin_0, end = var_2186_end_0, end_mask = var_2186_end_mask_0, squeeze_mask = var_2186_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_2186_cast_fp16")];
+            bool var_2187_transpose_x_0 = const()[name = string("op_2187_transpose_x_0"), val = bool(false)];
+            bool var_2187_transpose_y_0 = const()[name = string("op_2187_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2187_cast_fp16 = matmul(transpose_x = var_2187_transpose_x_0, transpose_y = var_2187_transpose_y_0, x = var_2186_cast_fp16, y = state_91_cast_fp16)[name = string("op_2187_cast_fp16")];
+            tensor<int32, [4]> var_2189_begin_0 = const()[name = string("op_2189_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_2189_end_0 = const()[name = string("op_2189_end_0"), val = tensor<int32, [4]>([32, 14, 64, 64])];
+            tensor<bool, [4]> var_2189_end_mask_0 = const()[name = string("op_2189_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2189_squeeze_mask_0 = const()[name = string("op_2189_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2189_cast_fp16 = slice_by_index(begin = var_2189_begin_0, end = var_2189_end_0, end_mask = var_2189_end_mask_0, squeeze_mask = var_2189_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_2189_cast_fp16")];
+            bool var_2190_transpose_x_0 = const()[name = string("op_2190_transpose_x_0"), val = bool(false)];
+            bool var_2190_transpose_y_0 = const()[name = string("op_2190_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2190_cast_fp16 = matmul(transpose_x = var_2190_transpose_x_0, transpose_y = var_2190_transpose_y_0, x = var_2189_cast_fp16, y = v_new_91_cast_fp16)[name = string("op_2190_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2191_cast_fp16 = add(x = var_2187_cast_fp16, y = var_2190_cast_fp16)[name = string("op_2191_cast_fp16")];
+            tensor<int32, [4]> var_2193_begin_0 = const()[name = string("op_2193_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_2193_end_0 = const()[name = string("op_2193_end_0"), val = tensor<int32, [4]>([32, 14, 1, 1])];
+            tensor<bool, [4]> var_2193_end_mask_0 = const()[name = string("op_2193_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2193_squeeze_mask_0 = const()[name = string("op_2193_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2193_cast_fp16 = slice_by_index(begin = var_2193_begin_0, end = var_2193_end_0, end_mask = var_2193_end_mask_0, squeeze_mask = var_2193_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_2193_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2194_cast_fp16 = mul(x = state_91_cast_fp16, y = var_2193_cast_fp16)[name = string("op_2194_cast_fp16")];
+            tensor<int32, [4]> var_2196_begin_0 = const()[name = string("op_2196_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_2196_end_0 = const()[name = string("op_2196_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_2196_end_mask_0 = const()[name = string("op_2196_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2196_squeeze_mask_0 = const()[name = string("op_2196_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2196_cast_fp16 = slice_by_index(begin = var_2196_begin_0, end = var_2196_end_0, end_mask = var_2196_end_mask_0, squeeze_mask = var_2196_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_2196_cast_fp16")];
+            bool var_2198_transpose_x_1 = const()[name = string("op_2198_transpose_x_1"), val = bool(true)];
+            bool var_2198_transpose_y_1 = const()[name = string("op_2198_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2198_cast_fp16 = matmul(transpose_x = var_2198_transpose_x_1, transpose_y = var_2198_transpose_y_1, x = var_2196_cast_fp16, y = v_new_91_cast_fp16)[name = string("op_2198_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_93_cast_fp16 = add(x = var_2194_cast_fp16, y = var_2198_cast_fp16)[name = string("state_93_cast_fp16")];
+            tensor<int32, [4]> var_2201_begin_0 = const()[name = string("op_2201_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_2201_end_0 = const()[name = string("op_2201_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_2201_end_mask_0 = const()[name = string("op_2201_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2201_squeeze_mask_0 = const()[name = string("op_2201_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2201_cast_fp16 = slice_by_index(begin = var_2201_begin_0, end = var_2201_end_0, end_mask = var_2201_end_mask_0, squeeze_mask = var_2201_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_2201_cast_fp16")];
+            tensor<int32, [4]> var_2203_begin_0 = const()[name = string("op_2203_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_2203_end_0 = const()[name = string("op_2203_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_2203_end_mask_0 = const()[name = string("op_2203_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2203_squeeze_mask_0 = const()[name = string("op_2203_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2203_cast_fp16 = slice_by_index(begin = var_2203_begin_0, end = var_2203_end_0, end_mask = var_2203_end_mask_0, squeeze_mask = var_2203_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_2203_cast_fp16")];
+            bool var_2204_transpose_x_0 = const()[name = string("op_2204_transpose_x_0"), val = bool(false)];
+            bool var_2204_transpose_y_0 = const()[name = string("op_2204_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2204_cast_fp16 = matmul(transpose_x = var_2204_transpose_x_0, transpose_y = var_2204_transpose_y_0, x = var_2203_cast_fp16, y = state_93_cast_fp16)[name = string("op_2204_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_93_cast_fp16 = sub(x = var_2201_cast_fp16, y = var_2204_cast_fp16)[name = string("v_new_93_cast_fp16")];
+            tensor<int32, [4]> var_2207_begin_0 = const()[name = string("op_2207_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_2207_end_0 = const()[name = string("op_2207_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_2207_end_mask_0 = const()[name = string("op_2207_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2207_squeeze_mask_0 = const()[name = string("op_2207_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2207_cast_fp16 = slice_by_index(begin = var_2207_begin_0, end = var_2207_end_0, end_mask = var_2207_end_mask_0, squeeze_mask = var_2207_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_2207_cast_fp16")];
+            bool var_2208_transpose_x_0 = const()[name = string("op_2208_transpose_x_0"), val = bool(false)];
+            bool var_2208_transpose_y_0 = const()[name = string("op_2208_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2208_cast_fp16 = matmul(transpose_x = var_2208_transpose_x_0, transpose_y = var_2208_transpose_y_0, x = var_2207_cast_fp16, y = state_93_cast_fp16)[name = string("op_2208_cast_fp16")];
+            tensor<int32, [4]> var_2210_begin_0 = const()[name = string("op_2210_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_2210_end_0 = const()[name = string("op_2210_end_0"), val = tensor<int32, [4]>([32, 15, 64, 64])];
+            tensor<bool, [4]> var_2210_end_mask_0 = const()[name = string("op_2210_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2210_squeeze_mask_0 = const()[name = string("op_2210_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2210_cast_fp16 = slice_by_index(begin = var_2210_begin_0, end = var_2210_end_0, end_mask = var_2210_end_mask_0, squeeze_mask = var_2210_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_2210_cast_fp16")];
+            bool var_2211_transpose_x_0 = const()[name = string("op_2211_transpose_x_0"), val = bool(false)];
+            bool var_2211_transpose_y_0 = const()[name = string("op_2211_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2211_cast_fp16 = matmul(transpose_x = var_2211_transpose_x_0, transpose_y = var_2211_transpose_y_0, x = var_2210_cast_fp16, y = v_new_93_cast_fp16)[name = string("op_2211_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2212_cast_fp16 = add(x = var_2208_cast_fp16, y = var_2211_cast_fp16)[name = string("op_2212_cast_fp16")];
+            tensor<int32, [4]> var_2214_begin_0 = const()[name = string("op_2214_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_2214_end_0 = const()[name = string("op_2214_end_0"), val = tensor<int32, [4]>([32, 15, 1, 1])];
+            tensor<bool, [4]> var_2214_end_mask_0 = const()[name = string("op_2214_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2214_squeeze_mask_0 = const()[name = string("op_2214_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2214_cast_fp16 = slice_by_index(begin = var_2214_begin_0, end = var_2214_end_0, end_mask = var_2214_end_mask_0, squeeze_mask = var_2214_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_2214_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2215_cast_fp16 = mul(x = state_93_cast_fp16, y = var_2214_cast_fp16)[name = string("op_2215_cast_fp16")];
+            tensor<int32, [4]> var_2217_begin_0 = const()[name = string("op_2217_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_2217_end_0 = const()[name = string("op_2217_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_2217_end_mask_0 = const()[name = string("op_2217_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2217_squeeze_mask_0 = const()[name = string("op_2217_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2217_cast_fp16 = slice_by_index(begin = var_2217_begin_0, end = var_2217_end_0, end_mask = var_2217_end_mask_0, squeeze_mask = var_2217_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_2217_cast_fp16")];
+            bool var_2219_transpose_x_1 = const()[name = string("op_2219_transpose_x_1"), val = bool(true)];
+            bool var_2219_transpose_y_1 = const()[name = string("op_2219_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2219_cast_fp16 = matmul(transpose_x = var_2219_transpose_x_1, transpose_y = var_2219_transpose_y_1, x = var_2217_cast_fp16, y = v_new_93_cast_fp16)[name = string("op_2219_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_95_cast_fp16 = add(x = var_2215_cast_fp16, y = var_2219_cast_fp16)[name = string("state_95_cast_fp16")];
+            tensor<int32, [4]> var_2222_begin_0 = const()[name = string("op_2222_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_2222_end_0 = const()[name = string("op_2222_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_2222_end_mask_0 = const()[name = string("op_2222_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2222_squeeze_mask_0 = const()[name = string("op_2222_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2222_cast_fp16 = slice_by_index(begin = var_2222_begin_0, end = var_2222_end_0, end_mask = var_2222_end_mask_0, squeeze_mask = var_2222_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_2222_cast_fp16")];
+            tensor<int32, [4]> var_2224_begin_0 = const()[name = string("op_2224_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_2224_end_0 = const()[name = string("op_2224_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_2224_end_mask_0 = const()[name = string("op_2224_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2224_squeeze_mask_0 = const()[name = string("op_2224_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2224_cast_fp16 = slice_by_index(begin = var_2224_begin_0, end = var_2224_end_0, end_mask = var_2224_end_mask_0, squeeze_mask = var_2224_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_2224_cast_fp16")];
+            bool var_2225_transpose_x_0 = const()[name = string("op_2225_transpose_x_0"), val = bool(false)];
+            bool var_2225_transpose_y_0 = const()[name = string("op_2225_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2225_cast_fp16 = matmul(transpose_x = var_2225_transpose_x_0, transpose_y = var_2225_transpose_y_0, x = var_2224_cast_fp16, y = state_95_cast_fp16)[name = string("op_2225_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_95_cast_fp16 = sub(x = var_2222_cast_fp16, y = var_2225_cast_fp16)[name = string("v_new_95_cast_fp16")];
+            tensor<int32, [4]> var_2228_begin_0 = const()[name = string("op_2228_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_2228_end_0 = const()[name = string("op_2228_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_2228_end_mask_0 = const()[name = string("op_2228_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2228_squeeze_mask_0 = const()[name = string("op_2228_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2228_cast_fp16 = slice_by_index(begin = var_2228_begin_0, end = var_2228_end_0, end_mask = var_2228_end_mask_0, squeeze_mask = var_2228_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_2228_cast_fp16")];
+            bool var_2229_transpose_x_0 = const()[name = string("op_2229_transpose_x_0"), val = bool(false)];
+            bool var_2229_transpose_y_0 = const()[name = string("op_2229_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2229_cast_fp16 = matmul(transpose_x = var_2229_transpose_x_0, transpose_y = var_2229_transpose_y_0, x = var_2228_cast_fp16, y = state_95_cast_fp16)[name = string("op_2229_cast_fp16")];
+            tensor<int32, [4]> var_2231_begin_0 = const()[name = string("op_2231_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_2231_end_0 = const()[name = string("op_2231_end_0"), val = tensor<int32, [4]>([32, 16, 64, 64])];
+            tensor<bool, [4]> var_2231_end_mask_0 = const()[name = string("op_2231_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2231_squeeze_mask_0 = const()[name = string("op_2231_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2231_cast_fp16 = slice_by_index(begin = var_2231_begin_0, end = var_2231_end_0, end_mask = var_2231_end_mask_0, squeeze_mask = var_2231_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_2231_cast_fp16")];
+            bool var_2232_transpose_x_0 = const()[name = string("op_2232_transpose_x_0"), val = bool(false)];
+            bool var_2232_transpose_y_0 = const()[name = string("op_2232_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2232_cast_fp16 = matmul(transpose_x = var_2232_transpose_x_0, transpose_y = var_2232_transpose_y_0, x = var_2231_cast_fp16, y = v_new_95_cast_fp16)[name = string("op_2232_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2233_cast_fp16 = add(x = var_2229_cast_fp16, y = var_2232_cast_fp16)[name = string("op_2233_cast_fp16")];
+            int32 var_2235_axis_0 = const()[name = string("op_2235_axis_0"), val = int32(1)];
+            tensor<fp16, [32, 16, 64, 128]> var_2235_cast_fp16 = stack(axis = var_2235_axis_0, values = (var_1918_cast_fp16, var_1939_cast_fp16, var_1960_cast_fp16, var_1981_cast_fp16, var_2002_cast_fp16, var_2023_cast_fp16, var_2044_cast_fp16, var_2065_cast_fp16, var_2086_cast_fp16, var_2107_cast_fp16, var_2128_cast_fp16, var_2149_cast_fp16, var_2170_cast_fp16, var_2191_cast_fp16, var_2212_cast_fp16, var_2233_cast_fp16))[name = string("op_2235_cast_fp16")];
+            tensor<int32, [3]> var_2236 = const()[name = string("op_2236"), val = tensor<int32, [3]>([32, 1024, 128])];
+            tensor<fp16, [32, 1024, 128]> var_2237_cast_fp16 = reshape(shape = var_2236, x = var_2235_cast_fp16)[name = string("op_2237_cast_fp16")];
+            tensor<int32, [3]> var_2238 = const()[name = string("op_2238"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> input_39_begin_0 = const()[name = string("input_39_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> input_39_end_0 = const()[name = string("input_39_end_0"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<bool, [4]> input_39_end_mask_0 = const()[name = string("input_39_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> input_39_squeeze_mask_0 = const()[name = string("input_39_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [1024, 32, 128]> input_39_cast_fp16 = slice_by_index(begin = input_39_begin_0, end = input_39_end_0, end_mask = input_39_end_mask_0, squeeze_mask = input_39_squeeze_mask_0, x = z_5_cast_fp16)[name = string("input_39_cast_fp16")];
+            fp16 var_1554_promoted_1_to_fp16 = const()[name = string("op_1554_promoted_1_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1024, 32, 128]> x_69_cast_fp16 = transpose(perm = var_2238, x = var_2237_cast_fp16)[name = string("transpose_32")];
+            tensor<fp16, [1024, 32, 128]> var_2242_cast_fp16 = pow(x = x_69_cast_fp16, y = var_1554_promoted_1_to_fp16)[name = string("op_2242_cast_fp16")];
+            tensor<int32, [1]> var_2244_axes_0 = const()[name = string("op_2244_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_2244_keep_dims_0 = const()[name = string("op_2244_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1024, 32, 1]> var_2244_cast_fp16 = reduce_mean(axes = var_2244_axes_0, keep_dims = var_2244_keep_dims_0, x = var_2242_cast_fp16)[name = string("op_2244_cast_fp16")];
+            fp16 var_2245_to_fp16 = const()[name = string("op_2245_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1024, 32, 1]> var_2246_cast_fp16 = add(x = var_2244_cast_fp16, y = var_2245_to_fp16)[name = string("op_2246_cast_fp16")];
+            fp32 var_2247_epsilon_0 = const()[name = string("op_2247_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1024, 32, 1]> var_2247_cast_fp16 = rsqrt(epsilon = var_2247_epsilon_0, x = var_2246_cast_fp16)[name = string("op_2247_cast_fp16")];
+            tensor<fp16, [1024, 32, 128]> x_71_cast_fp16 = mul(x = x_69_cast_fp16, y = var_2247_cast_fp16)[name = string("x_71_cast_fp16")];
+            tensor<fp16, [128]> layers_2_linear_attn_norm_weight_to_fp16 = const()[name = string("layers_2_linear_attn_norm_weight_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(685679808)))];
+            tensor<fp16, [1024, 32, 128]> var_2249_cast_fp16 = mul(x = layers_2_linear_attn_norm_weight_to_fp16, y = x_71_cast_fp16)[name = string("op_2249_cast_fp16")];
+            tensor<fp16, [1024, 32, 128]> var_2250_cast_fp16 = silu(x = input_39_cast_fp16)[name = string("op_2250_cast_fp16")];
+            tensor<fp16, [1024, 32, 128]> core_5_cast_fp16 = mul(x = var_2249_cast_fp16, y = var_2250_cast_fp16)[name = string("core_5_cast_fp16")];
+            tensor<int32, [3]> var_2252 = const()[name = string("op_2252"), val = tensor<int32, [3]>([1, 1024, 4096])];
+            tensor<fp16, [1, 1024, 4096]> input_41_cast_fp16 = reshape(shape = var_2252, x = core_5_cast_fp16)[name = string("input_41_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> linear_20_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers_2_linear_attn_out_proj_weight_cast_fp16, x = input_41_cast_fp16)[name = string("linear_20_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> x_73_cast_fp16 = add(x = x_51_cast_fp16, y = linear_20_cast_fp16)[name = string("x_73_cast_fp16")];
+            fp16 var_1554_promoted_2_to_fp16 = const()[name = string("op_1554_promoted_2_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 1024, 2560]> var_2258_cast_fp16 = pow(x = x_73_cast_fp16, y = var_1554_promoted_2_to_fp16)[name = string("op_2258_cast_fp16")];
+            tensor<int32, [1]> var_2260_axes_0 = const()[name = string("op_2260_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_2260_keep_dims_0 = const()[name = string("op_2260_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 1024, 1]> var_2260_cast_fp16 = reduce_mean(axes = var_2260_axes_0, keep_dims = var_2260_keep_dims_0, x = var_2258_cast_fp16)[name = string("op_2260_cast_fp16")];
+            fp16 var_2261_to_fp16 = const()[name = string("op_2261_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 1024, 1]> var_2262_cast_fp16 = add(x = var_2260_cast_fp16, y = var_2261_to_fp16)[name = string("op_2262_cast_fp16")];
+            fp32 var_2263_epsilon_0 = const()[name = string("op_2263_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 1024, 1]> var_2263_cast_fp16 = rsqrt(epsilon = var_2263_epsilon_0, x = var_2262_cast_fp16)[name = string("op_2263_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> x_75_cast_fp16 = mul(x = x_73_cast_fp16, y = var_2263_cast_fp16)[name = string("x_75_cast_fp16")];
+            tensor<fp16, [2560]> var_2266_to_fp16 = const()[name = string("op_2266_to_fp16"), val = tensor<fp16, [2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(685680128)))];
+            tensor<fp16, [1, 1024, 2560]> input_43_cast_fp16 = mul(x = x_75_cast_fp16, y = var_2266_to_fp16)[name = string("input_43_cast_fp16")];
+            tensor<fp16, [1, 1024, 9216]> linear_21_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = layers_2_mlp_gate_proj_weight_cast_fp16, x = input_43_cast_fp16)[name = string("linear_21_cast_fp16")];
+            tensor<fp16, [1, 1024, 9216]> var_2273_cast_fp16 = silu(x = linear_21_cast_fp16)[name = string("op_2273_cast_fp16")];
+            tensor<fp16, [1, 1024, 9216]> linear_22_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = layers_2_mlp_up_proj_weight_cast_fp16, x = input_43_cast_fp16)[name = string("linear_22_cast_fp16")];
+            tensor<fp16, [1, 1024, 9216]> input_47_cast_fp16 = mul(x = var_2273_cast_fp16, y = linear_22_cast_fp16)[name = string("input_47_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> linear_23_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers_2_mlp_down_proj_weight_cast_fp16, x = input_47_cast_fp16)[name = string("linear_23_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> x_77_cast_fp16 = add(x = x_73_cast_fp16, y = linear_23_cast_fp16)[name = string("x_77_cast_fp16")];
+            int32 var_2293 = const()[name = string("op_2293"), val = int32(-1)];
+            fp16 var_2292_promoted_to_fp16 = const()[name = string("op_2292_promoted_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 1024, 2560]> var_2302_cast_fp16 = pow(x = x_77_cast_fp16, y = var_2292_promoted_to_fp16)[name = string("op_2302_cast_fp16")];
+            tensor<int32, [1]> var_2304_axes_0 = const()[name = string("op_2304_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_2304_keep_dims_0 = const()[name = string("op_2304_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 1024, 1]> var_2304_cast_fp16 = reduce_mean(axes = var_2304_axes_0, keep_dims = var_2304_keep_dims_0, x = var_2302_cast_fp16)[name = string("op_2304_cast_fp16")];
+            fp16 var_2305_to_fp16 = const()[name = string("op_2305_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 1024, 1]> var_2306_cast_fp16 = add(x = var_2304_cast_fp16, y = var_2305_to_fp16)[name = string("op_2306_cast_fp16")];
+            fp32 var_2307_epsilon_0 = const()[name = string("op_2307_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 1024, 1]> var_2307_cast_fp16 = rsqrt(epsilon = var_2307_epsilon_0, x = var_2306_cast_fp16)[name = string("op_2307_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> x_79_cast_fp16 = mul(x = x_77_cast_fp16, y = var_2307_cast_fp16)[name = string("x_79_cast_fp16")];
+            tensor<fp16, [2560]> var_2310_to_fp16 = const()[name = string("op_2310_to_fp16"), val = tensor<fp16, [2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(685685312)))];
+            tensor<fp16, [1, 1024, 2560]> x_81_cast_fp16 = mul(x = x_79_cast_fp16, y = var_2310_to_fp16)[name = string("x_81_cast_fp16")];
+            tensor<fp16, [1, 1024, 8192]> linear_24_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = layers_3_self_attn_q_proj_weight_cast_fp16, x = x_81_cast_fp16)[name = string("linear_24_cast_fp16")];
+            tensor<int32, [4]> var_2321 = const()[name = string("op_2321"), val = tensor<int32, [4]>([1, 1024, 16, 512])];
+            tensor<fp16, [1, 1024, 16, 512]> qg_1_cast_fp16 = reshape(shape = var_2321, x = linear_24_cast_fp16)[name = string("qg_1_cast_fp16")];
+            tensor<int32, [4]> x_83_begin_0 = const()[name = string("x_83_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_83_end_0 = const()[name = string("x_83_end_0"), val = tensor<int32, [4]>([1, 1024, 16, 256])];
+            tensor<bool, [4]> x_83_end_mask_0 = const()[name = string("x_83_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1, 1024, 16, 256]> x_83_cast_fp16 = slice_by_index(begin = x_83_begin_0, end = x_83_end_0, end_mask = x_83_end_mask_0, x = qg_1_cast_fp16)[name = string("x_83_cast_fp16")];
+            tensor<int32, [4]> gate_1_begin_0 = const()[name = string("gate_1_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 256])];
+            tensor<int32, [4]> gate_1_end_0 = const()[name = string("gate_1_end_0"), val = tensor<int32, [4]>([1, 1024, 16, 512])];
+            tensor<bool, [4]> gate_1_end_mask_0 = const()[name = string("gate_1_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1, 1024, 16, 256]> gate_1_cast_fp16 = slice_by_index(begin = gate_1_begin_0, end = gate_1_end_0, end_mask = gate_1_end_mask_0, x = qg_1_cast_fp16)[name = string("gate_1_cast_fp16")];
+            tensor<int32, [3]> var_2325 = const()[name = string("op_2325"), val = tensor<int32, [3]>([1, 1024, 4096])];
+            tensor<fp16, [1, 1024, 4096]> gate_3_cast_fp16 = reshape(shape = var_2325, x = gate_1_cast_fp16)[name = string("gate_3_cast_fp16")];
+            fp16 var_2292_promoted_1_to_fp16 = const()[name = string("op_2292_promoted_1_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 1024, 16, 256]> var_2328_cast_fp16 = pow(x = x_83_cast_fp16, y = var_2292_promoted_1_to_fp16)[name = string("op_2328_cast_fp16")];
+            tensor<int32, [1]> var_2330_axes_0 = const()[name = string("op_2330_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_2330_keep_dims_0 = const()[name = string("op_2330_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 1024, 16, 1]> var_2330_cast_fp16 = reduce_mean(axes = var_2330_axes_0, keep_dims = var_2330_keep_dims_0, x = var_2328_cast_fp16)[name = string("op_2330_cast_fp16")];
+            fp16 var_2331_to_fp16 = const()[name = string("op_2331_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 1024, 16, 1]> var_2332_cast_fp16 = add(x = var_2330_cast_fp16, y = var_2331_to_fp16)[name = string("op_2332_cast_fp16")];
+            fp32 var_2333_epsilon_0 = const()[name = string("op_2333_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 1024, 16, 1]> var_2333_cast_fp16 = rsqrt(epsilon = var_2333_epsilon_0, x = var_2332_cast_fp16)[name = string("op_2333_cast_fp16")];
+            tensor<fp16, [1, 1024, 16, 256]> x_85_cast_fp16 = mul(x = x_83_cast_fp16, y = var_2333_cast_fp16)[name = string("x_85_cast_fp16")];
+            tensor<fp16, [256]> var_2336_to_fp16 = const()[name = string("op_2336_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(685690496)))];
+            tensor<fp16, [1, 1024, 16, 256]> var_2337_cast_fp16 = mul(x = x_85_cast_fp16, y = var_2336_to_fp16)[name = string("op_2337_cast_fp16")];
+            tensor<int32, [4]> q_31_perm_0 = const()[name = string("q_31_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
+            tensor<fp16, [1024]> linear_25_bias_0_to_fp16 = const()[name = string("linear_25_bias_0_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(685691072)))];
+            tensor<fp16, [1, 1024, 1024]> linear_25_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = layers_3_self_attn_k_proj_weight_cast_fp16, x = x_81_cast_fp16)[name = string("linear_25_cast_fp16")];
+            tensor<int32, [4]> var_2341 = const()[name = string("op_2341"), val = tensor<int32, [4]>([1, 1024, 4, 256])];
+            tensor<fp16, [1, 1024, 4, 256]> x_87_cast_fp16 = reshape(shape = var_2341, x = linear_25_cast_fp16)[name = string("x_87_cast_fp16")];
+            fp16 var_2292_promoted_2_to_fp16 = const()[name = string("op_2292_promoted_2_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 1024, 4, 256]> var_2344_cast_fp16 = pow(x = x_87_cast_fp16, y = var_2292_promoted_2_to_fp16)[name = string("op_2344_cast_fp16")];
+            tensor<int32, [1]> var_2346_axes_0 = const()[name = string("op_2346_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_2346_keep_dims_0 = const()[name = string("op_2346_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 1024, 4, 1]> var_2346_cast_fp16 = reduce_mean(axes = var_2346_axes_0, keep_dims = var_2346_keep_dims_0, x = var_2344_cast_fp16)[name = string("op_2346_cast_fp16")];
+            fp16 var_2347_to_fp16 = const()[name = string("op_2347_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 1024, 4, 1]> var_2348_cast_fp16 = add(x = var_2346_cast_fp16, y = var_2347_to_fp16)[name = string("op_2348_cast_fp16")];
+            fp32 var_2349_epsilon_0 = const()[name = string("op_2349_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 1024, 4, 1]> var_2349_cast_fp16 = rsqrt(epsilon = var_2349_epsilon_0, x = var_2348_cast_fp16)[name = string("op_2349_cast_fp16")];
+            tensor<fp16, [1, 1024, 4, 256]> x_89_cast_fp16 = mul(x = x_87_cast_fp16, y = var_2349_cast_fp16)[name = string("x_89_cast_fp16")];
+            tensor<fp16, [256]> var_2352_to_fp16 = const()[name = string("op_2352_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(685693184)))];
+            tensor<fp16, [1, 1024, 4, 256]> var_2353_cast_fp16 = mul(x = x_89_cast_fp16, y = var_2352_to_fp16)[name = string("op_2353_cast_fp16")];
+            tensor<int32, [4]> k_25_perm_0 = const()[name = string("k_25_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
+            tensor<fp16, [1, 1024, 1024]> linear_26_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = layers_3_self_attn_v_proj_weight_cast_fp16, x = x_81_cast_fp16)[name = string("linear_26_cast_fp16")];
+            tensor<int32, [4]> var_2357 = const()[name = string("op_2357"), val = tensor<int32, [4]>([1, 1024, 4, 256])];
+            tensor<fp16, [1, 1024, 4, 256]> var_2358_cast_fp16 = reshape(shape = var_2357, x = linear_26_cast_fp16)[name = string("op_2358_cast_fp16")];
+            tensor<int32, [4]> v_13_perm_0 = const()[name = string("v_13_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
+            tensor<int32, [1]> var_2360_axes_0 = const()[name = string("op_2360_axes_0"), val = tensor<int32, [1]>([0])];
+            tensor<fp16, [1, 1024, 64]> var_2360_cast_fp16 = expand_dims(axes = var_2360_axes_0, x = cos)[name = string("op_2360_cast_fp16")];
+            tensor<int32, [1]> c_1_axes_0 = const()[name = string("c_1_axes_0"), val = tensor<int32, [1]>([1])];
+            tensor<fp16, [1, 1, 1024, 64]> c_1_cast_fp16 = expand_dims(axes = c_1_axes_0, x = var_2360_cast_fp16)[name = string("c_1_cast_fp16")];
+            tensor<int32, [1]> var_2362_axes_0 = const()[name = string("op_2362_axes_0"), val = tensor<int32, [1]>([0])];
+            tensor<fp16, [1, 1024, 64]> var_2362_cast_fp16 = expand_dims(axes = var_2362_axes_0, x = sin)[name = string("op_2362_cast_fp16")];
+            tensor<int32, [1]> s_1_axes_0 = const()[name = string("s_1_axes_0"), val = tensor<int32, [1]>([1])];
+            tensor<fp16, [1, 1, 1024, 64]> s_1_cast_fp16 = expand_dims(axes = s_1_axes_0, x = var_2362_cast_fp16)[name = string("s_1_cast_fp16")];
+            tensor<int32, [4]> var_2364_begin_0 = const()[name = string("op_2364_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_2364_end_0 = const()[name = string("op_2364_end_0"), val = tensor<int32, [4]>([1, 16, 1024, 64])];
+            tensor<bool, [4]> var_2364_end_mask_0 = const()[name = string("op_2364_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1, 16, 1024, 256]> q_31_cast_fp16 = transpose(perm = q_31_perm_0, x = var_2337_cast_fp16)[name = string("transpose_31")];
+            tensor<fp16, [1, 16, 1024, 64]> var_2364_cast_fp16 = slice_by_index(begin = var_2364_begin_0, end = var_2364_end_0, end_mask = var_2364_end_mask_0, x = q_31_cast_fp16)[name = string("op_2364_cast_fp16")];
+            tensor<fp16, [1, 16, 1024, 64]> var_2365_cast_fp16 = mul(x = var_2364_cast_fp16, y = c_1_cast_fp16)[name = string("op_2365_cast_fp16")];
+            tensor<int32, [4]> var_2372_begin_0 = const()[name = string("op_2372_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 32])];
+            tensor<int32, [4]> var_2372_end_0 = const()[name = string("op_2372_end_0"), val = tensor<int32, [4]>([1, 16, 1024, 64])];
+            tensor<bool, [4]> var_2372_end_mask_0 = const()[name = string("op_2372_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1, 16, 1024, 32]> var_2372_cast_fp16 = slice_by_index(begin = var_2372_begin_0, end = var_2372_end_0, end_mask = var_2372_end_mask_0, x = var_2364_cast_fp16)[name = string("op_2372_cast_fp16")];
+            fp16 const_38_promoted_to_fp16 = const()[name = string("const_38_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1, 16, 1024, 32]> var_2373_cast_fp16 = mul(x = var_2372_cast_fp16, y = const_38_promoted_to_fp16)[name = string("op_2373_cast_fp16")];
+            tensor<int32, [4]> var_2374_begin_0 = const()[name = string("op_2374_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_2374_end_0 = const()[name = string("op_2374_end_0"), val = tensor<int32, [4]>([1, 16, 1024, 32])];
+            tensor<bool, [4]> var_2374_end_mask_0 = const()[name = string("op_2374_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1, 16, 1024, 32]> var_2374_cast_fp16 = slice_by_index(begin = var_2374_begin_0, end = var_2374_end_0, end_mask = var_2374_end_mask_0, x = var_2364_cast_fp16)[name = string("op_2374_cast_fp16")];
+            bool var_2376_interleave_0 = const()[name = string("op_2376_interleave_0"), val = bool(false)];
+            tensor<fp16, [1, 16, 1024, 64]> var_2376_cast_fp16 = concat(axis = var_2293, interleave = var_2376_interleave_0, values = (var_2373_cast_fp16, var_2374_cast_fp16))[name = string("op_2376_cast_fp16")];
+            tensor<fp16, [1, 16, 1024, 64]> var_2377_cast_fp16 = mul(x = var_2376_cast_fp16, y = s_1_cast_fp16)[name = string("op_2377_cast_fp16")];
+            tensor<fp16, [1, 16, 1024, 64]> var_2378_cast_fp16 = add(x = var_2365_cast_fp16, y = var_2377_cast_fp16)[name = string("op_2378_cast_fp16")];
+            tensor<int32, [4]> var_2379_begin_0 = const()[name = string("op_2379_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
+            tensor<int32, [4]> var_2379_end_0 = const()[name = string("op_2379_end_0"), val = tensor<int32, [4]>([1, 16, 1024, 256])];
+            tensor<bool, [4]> var_2379_end_mask_0 = const()[name = string("op_2379_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1, 16, 1024, 192]> var_2379_cast_fp16 = slice_by_index(begin = var_2379_begin_0, end = var_2379_end_0, end_mask = var_2379_end_mask_0, x = q_31_cast_fp16)[name = string("op_2379_cast_fp16")];
+            bool q_33_interleave_0 = const()[name = string("q_33_interleave_0"), val = bool(false)];
+            tensor<fp16, [1, 16, 1024, 256]> q_33_cast_fp16 = concat(axis = var_2293, interleave = q_33_interleave_0, values = (var_2378_cast_fp16, var_2379_cast_fp16))[name = string("q_33_cast_fp16")];
+            tensor<int32, [4]> var_2382_begin_0 = const()[name = string("op_2382_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_2382_end_0 = const()[name = string("op_2382_end_0"), val = tensor<int32, [4]>([1, 4, 1024, 64])];
+            tensor<bool, [4]> var_2382_end_mask_0 = const()[name = string("op_2382_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1, 4, 1024, 256]> k_25_cast_fp16 = transpose(perm = k_25_perm_0, x = var_2353_cast_fp16)[name = string("transpose_30")];
+            tensor<fp16, [1, 4, 1024, 64]> var_2382_cast_fp16 = slice_by_index(begin = var_2382_begin_0, end = var_2382_end_0, end_mask = var_2382_end_mask_0, x = k_25_cast_fp16)[name = string("op_2382_cast_fp16")];
+            tensor<fp16, [1, 4, 1024, 64]> var_2383_cast_fp16 = mul(x = var_2382_cast_fp16, y = c_1_cast_fp16)[name = string("op_2383_cast_fp16")];
+            tensor<int32, [4]> var_2390_begin_0 = const()[name = string("op_2390_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 32])];
+            tensor<int32, [4]> var_2390_end_0 = const()[name = string("op_2390_end_0"), val = tensor<int32, [4]>([1, 4, 1024, 64])];
+            tensor<bool, [4]> var_2390_end_mask_0 = const()[name = string("op_2390_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1, 4, 1024, 32]> var_2390_cast_fp16 = slice_by_index(begin = var_2390_begin_0, end = var_2390_end_0, end_mask = var_2390_end_mask_0, x = var_2382_cast_fp16)[name = string("op_2390_cast_fp16")];
+            fp16 const_40_promoted_to_fp16 = const()[name = string("const_40_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1, 4, 1024, 32]> var_2391_cast_fp16 = mul(x = var_2390_cast_fp16, y = const_40_promoted_to_fp16)[name = string("op_2391_cast_fp16")];
+            tensor<int32, [4]> var_2392_begin_0 = const()[name = string("op_2392_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_2392_end_0 = const()[name = string("op_2392_end_0"), val = tensor<int32, [4]>([1, 4, 1024, 32])];
+            tensor<bool, [4]> var_2392_end_mask_0 = const()[name = string("op_2392_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1, 4, 1024, 32]> var_2392_cast_fp16 = slice_by_index(begin = var_2392_begin_0, end = var_2392_end_0, end_mask = var_2392_end_mask_0, x = var_2382_cast_fp16)[name = string("op_2392_cast_fp16")];
+            bool var_2394_interleave_0 = const()[name = string("op_2394_interleave_0"), val = bool(false)];
+            tensor<fp16, [1, 4, 1024, 64]> var_2394_cast_fp16 = concat(axis = var_2293, interleave = var_2394_interleave_0, values = (var_2391_cast_fp16, var_2392_cast_fp16))[name = string("op_2394_cast_fp16")];
+            tensor<fp16, [1, 4, 1024, 64]> var_2395_cast_fp16 = mul(x = var_2394_cast_fp16, y = s_1_cast_fp16)[name = string("op_2395_cast_fp16")];
+            tensor<fp16, [1, 4, 1024, 64]> var_2396_cast_fp16 = add(x = var_2383_cast_fp16, y = var_2395_cast_fp16)[name = string("op_2396_cast_fp16")];
+            tensor<int32, [4]> var_2397_begin_0 = const()[name = string("op_2397_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
+            tensor<int32, [4]> var_2397_end_0 = const()[name = string("op_2397_end_0"), val = tensor<int32, [4]>([1, 4, 1024, 256])];
+            tensor<bool, [4]> var_2397_end_mask_0 = const()[name = string("op_2397_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1, 4, 1024, 192]> var_2397_cast_fp16 = slice_by_index(begin = var_2397_begin_0, end = var_2397_end_0, end_mask = var_2397_end_mask_0, x = k_25_cast_fp16)[name = string("op_2397_cast_fp16")];
+            bool k_27_interleave_0 = const()[name = string("k_27_interleave_0"), val = bool(false)];
+            tensor<fp16, [1, 4, 1024, 256]> k_27_cast_fp16 = concat(axis = var_2293, interleave = k_27_interleave_0, values = (var_2396_cast_fp16, var_2397_cast_fp16))[name = string("k_27_cast_fp16")];
+            tensor<int32, [1]> var_2402_axes_0 = const()[name = string("op_2402_axes_0"), val = tensor<int32, [1]>([2])];
+            tensor<fp16, [1, 4, 1, 1024, 256]> var_2402_cast_fp16 = expand_dims(axes = var_2402_axes_0, x = k_27_cast_fp16)[name = string("op_2402_cast_fp16")];
+            tensor<int32, [5]> var_2404_reps_0 = const()[name = string("op_2404_reps_0"), val = tensor<int32, [5]>([1, 1, 4, 1, 1])];
+            tensor<fp16, [1, 4, 4, 1024, 256]> var_2404_cast_fp16 = tile(reps = var_2404_reps_0, x = var_2402_cast_fp16)[name = string("op_2404_cast_fp16")];
+            tensor<int32, [4]> var_2405 = const()[name = string("op_2405"), val = tensor<int32, [4]>([1, 16, 1024, 256])];
+            tensor<fp16, [1, 16, 1024, 256]> k_29_cast_fp16 = reshape(shape = var_2405, x = var_2404_cast_fp16)[name = string("k_29_cast_fp16")];
+            tensor<int32, [1]> var_2409_axes_0 = const()[name = string("op_2409_axes_0"), val = tensor<int32, [1]>([2])];
+            tensor<fp16, [1, 4, 1024, 256]> v_13_cast_fp16 = transpose(perm = v_13_perm_0, x = var_2358_cast_fp16)[name = string("transpose_29")];
+            tensor<fp16, [1, 4, 1, 1024, 256]> var_2409_cast_fp16 = expand_dims(axes = var_2409_axes_0, x = v_13_cast_fp16)[name = string("op_2409_cast_fp16")];
+            tensor<int32, [5]> var_2411_reps_0 = const()[name = string("op_2411_reps_0"), val = tensor<int32, [5]>([1, 1, 4, 1, 1])];
+            tensor<fp16, [1, 4, 4, 1024, 256]> var_2411_cast_fp16 = tile(reps = var_2411_reps_0, x = var_2409_cast_fp16)[name = string("op_2411_cast_fp16")];
+            tensor<int32, [4]> var_2412 = const()[name = string("op_2412"), val = tensor<int32, [4]>([1, 16, 1024, 256])];
+            tensor<fp16, [1, 16, 1024, 256]> v_15_cast_fp16 = reshape(shape = var_2412, x = var_2411_cast_fp16)[name = string("v_15_cast_fp16")];
+            bool var_2415_transpose_x_1 = const()[name = string("op_2415_transpose_x_1"), val = bool(false)];
+            bool var_2415_transpose_y_1 = const()[name = string("op_2415_transpose_y_1"), val = bool(true)];
+            tensor<fp16, [1, 16, 1024, 1024]> var_2415_cast_fp16 = matmul(transpose_x = var_2415_transpose_x_1, transpose_y = var_2415_transpose_y_1, x = q_33_cast_fp16, y = k_29_cast_fp16)[name = string("op_2415_cast_fp16")];
+            fp16 var_2416_to_fp16 = const()[name = string("op_2416_to_fp16"), val = fp16(0x1p-4)];
+            tensor<fp16, [1, 16, 1024, 1024]> var_2417_cast_fp16 = mul(x = var_2415_cast_fp16, y = var_2416_to_fp16)[name = string("op_2417_cast_fp16")];
+            tensor<fp16, [1, 1, 1024, 1024]> mask_to_fp16 = const()[name = string("mask_to_fp16"), val = tensor<fp16, [1, 1, 1024, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(685693760)))];
+            tensor<fp16, [1, 16, 1024, 1024]> scores_1_cast_fp16 = add(x = var_2417_cast_fp16, y = mask_to_fp16)[name = string("scores_1_cast_fp16")];
+            tensor<fp16, [1, 16, 1024, 1024]> var_2419_cast_fp16 = softmax(axis = var_2293, x = scores_1_cast_fp16)[name = string("op_2419_cast_fp16")];
+            bool out_1_transpose_x_0 = const()[name = string("out_1_transpose_x_0"), val = bool(false)];
+            bool out_1_transpose_y_0 = const()[name = string("out_1_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1, 16, 1024, 256]> out_1_cast_fp16 = matmul(transpose_x = out_1_transpose_x_0, transpose_y = out_1_transpose_y_0, x = var_2419_cast_fp16, y = v_15_cast_fp16)[name = string("out_1_cast_fp16")];
+            tensor<int32, [4]> var_2421_perm_0 = const()[name = string("op_2421_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
+            tensor<int32, [3]> var_2422 = const()[name = string("op_2422"), val = tensor<int32, [3]>([1, 1024, 4096])];
+            tensor<fp16, [1, 1024, 16, 256]> var_2421_cast_fp16 = transpose(perm = var_2421_perm_0, x = out_1_cast_fp16)[name = string("transpose_28")];
+            tensor<fp16, [1, 1024, 4096]> out_3_cast_fp16 = reshape(shape = var_2422, x = var_2421_cast_fp16)[name = string("out_3_cast_fp16")];
+            tensor<fp16, [1, 1024, 4096]> var_2424_cast_fp16 = sigmoid(x = gate_3_cast_fp16)[name = string("op_2424_cast_fp16")];
+            tensor<fp16, [1, 1024, 4096]> input_49_cast_fp16 = mul(x = out_3_cast_fp16, y = var_2424_cast_fp16)[name = string("input_49_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> linear_27_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers_3_self_attn_o_proj_weight_cast_fp16, x = input_49_cast_fp16)[name = string("linear_27_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> x_95_cast_fp16 = add(x = x_77_cast_fp16, y = linear_27_cast_fp16)[name = string("x_95_cast_fp16")];
+            fp16 var_2292_promoted_3_to_fp16 = const()[name = string("op_2292_promoted_3_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 1024, 2560]> var_2430_cast_fp16 = pow(x = x_95_cast_fp16, y = var_2292_promoted_3_to_fp16)[name = string("op_2430_cast_fp16")];
+            tensor<int32, [1]> var_2432_axes_0 = const()[name = string("op_2432_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_2432_keep_dims_0 = const()[name = string("op_2432_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 1024, 1]> var_2432_cast_fp16 = reduce_mean(axes = var_2432_axes_0, keep_dims = var_2432_keep_dims_0, x = var_2430_cast_fp16)[name = string("op_2432_cast_fp16")];
+            fp16 var_2433_to_fp16 = const()[name = string("op_2433_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 1024, 1]> var_2434_cast_fp16 = add(x = var_2432_cast_fp16, y = var_2433_to_fp16)[name = string("op_2434_cast_fp16")];
+            fp32 var_2435_epsilon_0 = const()[name = string("op_2435_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 1024, 1]> var_2435_cast_fp16 = rsqrt(epsilon = var_2435_epsilon_0, x = var_2434_cast_fp16)[name = string("op_2435_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> x_97_cast_fp16 = mul(x = x_95_cast_fp16, y = var_2435_cast_fp16)[name = string("x_97_cast_fp16")];
+            tensor<fp16, [2560]> var_2438_to_fp16 = const()[name = string("op_2438_to_fp16"), val = tensor<fp16, [2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(687790976)))];
+            tensor<fp16, [1, 1024, 2560]> input_51_cast_fp16 = mul(x = x_97_cast_fp16, y = var_2438_to_fp16)[name = string("input_51_cast_fp16")];
+            tensor<fp16, [1, 1024, 9216]> linear_28_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = layers_3_mlp_gate_proj_weight_cast_fp16, x = input_51_cast_fp16)[name = string("linear_28_cast_fp16")];
+            tensor<fp16, [1, 1024, 9216]> var_2445_cast_fp16 = silu(x = linear_28_cast_fp16)[name = string("op_2445_cast_fp16")];
+            tensor<fp16, [1, 1024, 9216]> linear_29_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = layers_3_mlp_up_proj_weight_cast_fp16, x = input_51_cast_fp16)[name = string("linear_29_cast_fp16")];
+            tensor<fp16, [1, 1024, 9216]> input_55_cast_fp16 = mul(x = var_2445_cast_fp16, y = linear_29_cast_fp16)[name = string("input_55_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> linear_30_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers_3_mlp_down_proj_weight_cast_fp16, x = input_55_cast_fp16)[name = string("linear_30_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> x_99_cast_fp16 = add(x = x_95_cast_fp16, y = linear_30_cast_fp16)[name = string("x_99_cast_fp16")];
+            int32 var_2465 = const()[name = string("op_2465"), val = int32(-2)];
+            int32 var_2480 = const()[name = string("op_2480"), val = int32(-1)];
+            fp16 var_2479_promoted_to_fp16 = const()[name = string("op_2479_promoted_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 1024, 2560]> var_2489_cast_fp16 = pow(x = x_99_cast_fp16, y = var_2479_promoted_to_fp16)[name = string("op_2489_cast_fp16")];
+            tensor<int32, [1]> var_2491_axes_0 = const()[name = string("op_2491_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_2491_keep_dims_0 = const()[name = string("op_2491_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 1024, 1]> var_2491_cast_fp16 = reduce_mean(axes = var_2491_axes_0, keep_dims = var_2491_keep_dims_0, x = var_2489_cast_fp16)[name = string("op_2491_cast_fp16")];
+            fp16 var_2492_to_fp16 = const()[name = string("op_2492_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 1024, 1]> var_2493_cast_fp16 = add(x = var_2491_cast_fp16, y = var_2492_to_fp16)[name = string("op_2493_cast_fp16")];
+            fp32 var_2494_epsilon_0 = const()[name = string("op_2494_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 1024, 1]> var_2494_cast_fp16 = rsqrt(epsilon = var_2494_epsilon_0, x = var_2493_cast_fp16)[name = string("op_2494_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> x_101_cast_fp16 = mul(x = x_99_cast_fp16, y = var_2494_cast_fp16)[name = string("x_101_cast_fp16")];
+            tensor<fp16, [2560]> var_2497_to_fp16 = const()[name = string("op_2497_to_fp16"), val = tensor<fp16, [2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(687796160)))];
+            tensor<fp16, [1, 1024, 2560]> x_103_cast_fp16 = mul(x = x_101_cast_fp16, y = var_2497_to_fp16)[name = string("x_103_cast_fp16")];
+            tensor<fp16, [1, 1024, 8192]> linear_31_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = layers_4_linear_attn_in_proj_qkv_weight_cast_fp16, x = x_103_cast_fp16)[name = string("linear_31_cast_fp16")];
+            tensor<int32, [3]> input_57_perm_0 = const()[name = string("input_57_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
+            tensor<int32, [6]> input_59_pad_0 = const()[name = string("input_59_pad_0"), val = tensor<int32, [6]>([0, 0, 0, 0, 3, 0])];
+            string input_59_mode_0 = const()[name = string("input_59_mode_0"), val = string("constant")];
+            fp16 const_42_to_fp16 = const()[name = string("const_42_to_fp16"), val = fp16(0x0p+0)];
+            tensor<fp16, [1, 8192, 1024]> input_57_cast_fp16 = transpose(perm = input_57_perm_0, x = linear_31_cast_fp16)[name = string("transpose_27")];
+            tensor<fp16, [1, 8192, 1027]> input_59_cast_fp16 = pad(constant_val = const_42_to_fp16, mode = input_59_mode_0, pad = input_59_pad_0, x = input_57_cast_fp16)[name = string("input_59_cast_fp16")];
+            string input_61_pad_type_0 = const()[name = string("input_61_pad_type_0"), val = string("valid")];
+            int32 input_61_groups_0 = const()[name = string("input_61_groups_0"), val = int32(8192)];
+            tensor<int32, [1]> input_61_strides_0 = const()[name = string("input_61_strides_0"), val = tensor<int32, [1]>([1])];
+            tensor<int32, [2]> input_61_pad_0 = const()[name = string("input_61_pad_0"), val = tensor<int32, [2]>([0, 0])];
+            tensor<int32, [1]> input_61_dilations_0 = const()[name = string("input_61_dilations_0"), val = tensor<int32, [1]>([1])];
+            tensor<fp16, [8192, 1, 4]> layers_4_linear_attn_conv1d_weight_to_fp16 = const()[name = string("layers_4_linear_attn_conv1d_weight_to_fp16"), val = tensor<fp16, [8192, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(687801344)))];
+            tensor<fp16, [1, 8192, 1024]> input_61_cast_fp16 = conv(dilations = input_61_dilations_0, groups = input_61_groups_0, pad = input_61_pad_0, pad_type = input_61_pad_type_0, strides = input_61_strides_0, weight = layers_4_linear_attn_conv1d_weight_to_fp16, x = input_59_cast_fp16)[name = string("input_61_cast_fp16")];
+            tensor<fp16, [1, 8192, 1024]> var_2532_cast_fp16 = silu(x = input_61_cast_fp16)[name = string("op_2532_cast_fp16")];
+            tensor<int32, [3]> qkv_7_perm_0 = const()[name = string("qkv_7_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
+            tensor<int32, [3]> var_2534_begin_0 = const()[name = string("op_2534_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_2534_end_0 = const()[name = string("op_2534_end_0"), val = tensor<int32, [3]>([1, 1024, 2048])];
+            tensor<bool, [3]> var_2534_end_mask_0 = const()[name = string("op_2534_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
+            tensor<fp16, [1, 1024, 8192]> qkv_7_cast_fp16 = transpose(perm = qkv_7_perm_0, x = var_2532_cast_fp16)[name = string("transpose_26")];
+            tensor<fp16, [1, 1024, 2048]> var_2534_cast_fp16 = slice_by_index(begin = var_2534_begin_0, end = var_2534_end_0, end_mask = var_2534_end_mask_0, x = qkv_7_cast_fp16)[name = string("op_2534_cast_fp16")];
+            tensor<int32, [4]> var_2535 = const()[name = string("op_2535"), val = tensor<int32, [4]>([1, 1024, 16, 128])];
+            tensor<fp16, [1, 1024, 16, 128]> q_35_cast_fp16 = reshape(shape = var_2535, x = var_2534_cast_fp16)[name = string("q_35_cast_fp16")];
+            tensor<int32, [3]> var_2537_begin_0 = const()[name = string("op_2537_begin_0"), val = tensor<int32, [3]>([0, 0, 2048])];
+            tensor<int32, [3]> var_2537_end_0 = const()[name = string("op_2537_end_0"), val = tensor<int32, [3]>([1, 1024, 4096])];
+            tensor<bool, [3]> var_2537_end_mask_0 = const()[name = string("op_2537_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
+            tensor<fp16, [1, 1024, 2048]> var_2537_cast_fp16 = slice_by_index(begin = var_2537_begin_0, end = var_2537_end_0, end_mask = var_2537_end_mask_0, x = qkv_7_cast_fp16)[name = string("op_2537_cast_fp16")];
+            tensor<int32, [4]> var_2538 = const()[name = string("op_2538"), val = tensor<int32, [4]>([1, 1024, 16, 128])];
+            tensor<fp16, [1, 1024, 16, 128]> k_31_cast_fp16 = reshape(shape = var_2538, x = var_2537_cast_fp16)[name = string("k_31_cast_fp16")];
+            tensor<int32, [3]> var_2540_begin_0 = const()[name = string("op_2540_begin_0"), val = tensor<int32, [3]>([0, 0, 4096])];
+            tensor<int32, [3]> var_2540_end_0 = const()[name = string("op_2540_end_0"), val = tensor<int32, [3]>([1, 1024, 8192])];
+            tensor<bool, [3]> var_2540_end_mask_0 = const()[name = string("op_2540_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
+            tensor<fp16, [1, 1024, 4096]> var_2540_cast_fp16 = slice_by_index(begin = var_2540_begin_0, end = var_2540_end_0, end_mask = var_2540_end_mask_0, x = qkv_7_cast_fp16)[name = string("op_2540_cast_fp16")];
+            tensor<int32, [4]> var_2541 = const()[name = string("op_2541"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<fp16, [1, 1024, 32, 128]> v_17_cast_fp16 = reshape(shape = var_2541, x = var_2540_cast_fp16)[name = string("v_17_cast_fp16")];
+            tensor<fp16, [1, 1024, 4096]> linear_32_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = layers_4_linear_attn_in_proj_z_weight_cast_fp16, x = x_103_cast_fp16)[name = string("linear_32_cast_fp16")];
+            tensor<int32, [4]> var_2545 = const()[name = string("op_2545"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<fp16, [1, 1024, 32, 128]> z_7_cast_fp16 = reshape(shape = var_2545, x = linear_32_cast_fp16)[name = string("z_7_cast_fp16")];
+            tensor<fp16, [32]> linear_33_bias_0_to_fp16 = const()[name = string("linear_33_bias_0_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(687866944)))];
+            tensor<fp16, [1, 1024, 32]> linear_33_cast_fp16 = linear(bias = linear_33_bias_0_to_fp16, weight = layers_4_linear_attn_in_proj_b_weight_cast_fp16, x = x_103_cast_fp16)[name = string("linear_33_cast_fp16")];
+            tensor<fp16, [1, 1024, 32]> beta_13_cast_fp16 = sigmoid(x = linear_33_cast_fp16)[name = string("beta_13_cast_fp16")];
+            tensor<fp16, [32]> linear_34_bias_0_to_fp16 = const()[name = string("linear_34_bias_0_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(687867072)))];
+            tensor<fp16, [1, 1024, 32]> linear_34_cast_fp16 = linear(bias = linear_34_bias_0_to_fp16, weight = layers_4_linear_attn_in_proj_a_weight_cast_fp16, x = x_103_cast_fp16)[name = string("linear_34_cast_fp16")];
+            tensor<fp16, [32]> layers_4_linear_attn_dt_bias_to_fp16 = const()[name = string("layers_4_linear_attn_dt_bias_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(687867200)))];
+            tensor<fp16, [1, 1024, 32]> var_2554_cast_fp16 = add(x = linear_34_cast_fp16, y = layers_4_linear_attn_dt_bias_to_fp16)[name = string("op_2554_cast_fp16")];
+            tensor<fp16, [1, 1024, 32]> var_2555_cast_fp16 = softplus(x = var_2554_cast_fp16)[name = string("op_2555_cast_fp16")];
+            tensor<fp16, [32]> var_2551_to_fp16 = const()[name = string("op_2551_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(687867328)))];
+            tensor<fp16, [1, 1024, 32]> g_13_cast_fp16 = mul(x = var_2551_to_fp16, y = var_2555_cast_fp16)[name = string("g_13_cast_fp16")];
+            tensor<fp16, [1, 1024, 16, 128]> var_2557_cast_fp16 = mul(x = q_35_cast_fp16, y = q_35_cast_fp16)[name = string("op_2557_cast_fp16")];
+            tensor<int32, [1]> var_2559_axes_0 = const()[name = string("op_2559_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_2559_keep_dims_0 = const()[name = string("op_2559_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 1024, 16, 1]> var_2559_cast_fp16 = reduce_sum(axes = var_2559_axes_0, keep_dims = var_2559_keep_dims_0, x = var_2557_cast_fp16)[name = string("op_2559_cast_fp16")];
+            fp16 var_2560_to_fp16 = const()[name = string("op_2560_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 1024, 16, 1]> var_2561_cast_fp16 = add(x = var_2559_cast_fp16, y = var_2560_to_fp16)[name = string("op_2561_cast_fp16")];
+            fp32 var_2562_epsilon_0 = const()[name = string("op_2562_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 1024, 16, 1]> var_2562_cast_fp16 = rsqrt(epsilon = var_2562_epsilon_0, x = var_2561_cast_fp16)[name = string("op_2562_cast_fp16")];
+            tensor<fp16, [1, 1024, 16, 128]> q_37_cast_fp16 = mul(x = q_35_cast_fp16, y = var_2562_cast_fp16)[name = string("q_37_cast_fp16")];
+            tensor<fp16, [1, 1024, 16, 128]> var_2564_cast_fp16 = mul(x = k_31_cast_fp16, y = k_31_cast_fp16)[name = string("op_2564_cast_fp16")];
+            tensor<int32, [1]> var_2566_axes_0 = const()[name = string("op_2566_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_2566_keep_dims_0 = const()[name = string("op_2566_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 1024, 16, 1]> var_2566_cast_fp16 = reduce_sum(axes = var_2566_axes_0, keep_dims = var_2566_keep_dims_0, x = var_2564_cast_fp16)[name = string("op_2566_cast_fp16")];
+            fp16 var_2567_to_fp16 = const()[name = string("op_2567_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 1024, 16, 1]> var_2568_cast_fp16 = add(x = var_2566_cast_fp16, y = var_2567_to_fp16)[name = string("op_2568_cast_fp16")];
+            fp32 var_2569_epsilon_0 = const()[name = string("op_2569_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 1024, 16, 1]> var_2569_cast_fp16 = rsqrt(epsilon = var_2569_epsilon_0, x = var_2568_cast_fp16)[name = string("op_2569_cast_fp16")];
+            tensor<fp16, [1, 1024, 16, 128]> k_33_cast_fp16 = mul(x = k_31_cast_fp16, y = var_2569_cast_fp16)[name = string("k_33_cast_fp16")];
+            tensor<int32, [1]> var_2574_axes_0 = const()[name = string("op_2574_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [1, 1024, 16, 1, 128]> var_2574_cast_fp16 = expand_dims(axes = var_2574_axes_0, x = q_37_cast_fp16)[name = string("op_2574_cast_fp16")];
+            tensor<int32, [5]> var_2576_reps_0 = const()[name = string("op_2576_reps_0"), val = tensor<int32, [5]>([1, 1, 1, 2, 1])];
+            tensor<fp16, [1, 1024, 16, 2, 128]> var_2576_cast_fp16 = tile(reps = var_2576_reps_0, x = var_2574_cast_fp16)[name = string("op_2576_cast_fp16")];
+            tensor<int32, [4]> var_2577 = const()[name = string("op_2577"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<fp16, [1, 1024, 32, 128]> q_39_cast_fp16 = reshape(shape = var_2577, x = var_2576_cast_fp16)[name = string("q_39_cast_fp16")];
+            tensor<int32, [1]> var_2582_axes_0 = const()[name = string("op_2582_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [1, 1024, 16, 1, 128]> var_2582_cast_fp16 = expand_dims(axes = var_2582_axes_0, x = k_33_cast_fp16)[name = string("op_2582_cast_fp16")];
+            tensor<int32, [5]> var_2584_reps_0 = const()[name = string("op_2584_reps_0"), val = tensor<int32, [5]>([1, 1, 1, 2, 1])];
+            tensor<fp16, [1, 1024, 16, 2, 128]> var_2584_cast_fp16 = tile(reps = var_2584_reps_0, x = var_2582_cast_fp16)[name = string("op_2584_cast_fp16")];
+            tensor<int32, [4]> var_2585 = const()[name = string("op_2585"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<fp16, [1, 1024, 32, 128]> k_35_cast_fp16 = reshape(shape = var_2585, x = var_2584_cast_fp16)[name = string("k_35_cast_fp16")];
+            fp16 var_2587_to_fp16 = const()[name = string("op_2587_to_fp16"), val = fp16(0x1.6ap-4)];
+            tensor<fp16, [1, 1024, 32, 128]> q_41_cast_fp16 = mul(x = q_39_cast_fp16, y = var_2587_to_fp16)[name = string("q_41_cast_fp16")];
+            tensor<int32, [4]> var_2589_begin_0 = const()[name = string("op_2589_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_2589_end_0 = const()[name = string("op_2589_end_0"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<bool, [4]> var_2589_end_mask_0 = const()[name = string("op_2589_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_2589_squeeze_mask_0 = const()[name = string("op_2589_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [1024, 32, 128]> var_2589_cast_fp16 = slice_by_index(begin = var_2589_begin_0, end = var_2589_end_0, end_mask = var_2589_end_mask_0, squeeze_mask = var_2589_squeeze_mask_0, x = q_41_cast_fp16)[name = string("op_2589_cast_fp16")];
+            tensor<int32, [3]> var_2590 = const()[name = string("op_2590"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_2592 = const()[name = string("op_2592"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<fp16, [32, 1024, 128]> var_2591_cast_fp16 = transpose(perm = var_2590, x = var_2589_cast_fp16)[name = string("transpose_25")];
+            tensor<fp16, [32, 16, 64, 128]> q_43_cast_fp16 = reshape(shape = var_2592, x = var_2591_cast_fp16)[name = string("q_43_cast_fp16")];
+            tensor<int32, [4]> var_2594_begin_0 = const()[name = string("op_2594_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_2594_end_0 = const()[name = string("op_2594_end_0"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<bool, [4]> var_2594_end_mask_0 = const()[name = string("op_2594_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_2594_squeeze_mask_0 = const()[name = string("op_2594_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [1024, 32, 128]> var_2594_cast_fp16 = slice_by_index(begin = var_2594_begin_0, end = var_2594_end_0, end_mask = var_2594_end_mask_0, squeeze_mask = var_2594_squeeze_mask_0, x = k_35_cast_fp16)[name = string("op_2594_cast_fp16")];
+            tensor<int32, [3]> var_2595 = const()[name = string("op_2595"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_2597 = const()[name = string("op_2597"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<fp16, [32, 1024, 128]> var_2596_cast_fp16 = transpose(perm = var_2595, x = var_2594_cast_fp16)[name = string("transpose_24")];
+            tensor<fp16, [32, 16, 64, 128]> k_37_cast_fp16 = reshape(shape = var_2597, x = var_2596_cast_fp16)[name = string("k_37_cast_fp16")];
+            tensor<int32, [4]> var_2599_begin_0 = const()[name = string("op_2599_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_2599_end_0 = const()[name = string("op_2599_end_0"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<bool, [4]> var_2599_end_mask_0 = const()[name = string("op_2599_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_2599_squeeze_mask_0 = const()[name = string("op_2599_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [1024, 32, 128]> var_2599_cast_fp16 = slice_by_index(begin = var_2599_begin_0, end = var_2599_end_0, end_mask = var_2599_end_mask_0, squeeze_mask = var_2599_squeeze_mask_0, x = v_17_cast_fp16)[name = string("op_2599_cast_fp16")];
+            tensor<int32, [3]> var_2600 = const()[name = string("op_2600"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_2602 = const()[name = string("op_2602"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<fp16, [32, 1024, 128]> var_2601_cast_fp16 = transpose(perm = var_2600, x = var_2599_cast_fp16)[name = string("transpose_23")];
+            tensor<fp16, [32, 16, 64, 128]> v_19_cast_fp16 = reshape(shape = var_2602, x = var_2601_cast_fp16)[name = string("v_19_cast_fp16")];
+            tensor<int32, [3]> var_2604_begin_0 = const()[name = string("op_2604_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_2604_end_0 = const()[name = string("op_2604_end_0"), val = tensor<int32, [3]>([1, 1024, 32])];
+            tensor<bool, [3]> var_2604_end_mask_0 = const()[name = string("op_2604_end_mask_0"), val = tensor<bool, [3]>([false, true, true])];
+            tensor<bool, [3]> var_2604_squeeze_mask_0 = const()[name = string("op_2604_squeeze_mask_0"), val = tensor<bool, [3]>([true, false, false])];
+            tensor<fp16, [1024, 32]> var_2604_cast_fp16 = slice_by_index(begin = var_2604_begin_0, end = var_2604_end_0, end_mask = var_2604_end_mask_0, squeeze_mask = var_2604_squeeze_mask_0, x = beta_13_cast_fp16)[name = string("op_2604_cast_fp16")];
+            tensor<int32, [2]> var_2605_perm_0 = const()[name = string("op_2605_perm_0"), val = tensor<int32, [2]>([1, 0])];
+            tensor<int32, [4]> var_2606 = const()[name = string("op_2606"), val = tensor<int32, [4]>([32, 16, 64, 1])];
+            tensor<fp16, [32, 1024]> var_2605_cast_fp16 = transpose(perm = var_2605_perm_0, x = var_2604_cast_fp16)[name = string("transpose_22")];
+            tensor<fp16, [32, 16, 64, 1]> beta_15_cast_fp16 = reshape(shape = var_2606, x = var_2605_cast_fp16)[name = string("beta_15_cast_fp16")];
+            tensor<int32, [3]> var_2608_begin_0 = const()[name = string("op_2608_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_2608_end_0 = const()[name = string("op_2608_end_0"), val = tensor<int32, [3]>([1, 1024, 32])];
+            tensor<bool, [3]> var_2608_end_mask_0 = const()[name = string("op_2608_end_mask_0"), val = tensor<bool, [3]>([false, true, true])];
+            tensor<bool, [3]> var_2608_squeeze_mask_0 = const()[name = string("op_2608_squeeze_mask_0"), val = tensor<bool, [3]>([true, false, false])];
+            tensor<fp16, [1024, 32]> var_2608_cast_fp16 = slice_by_index(begin = var_2608_begin_0, end = var_2608_end_0, end_mask = var_2608_end_mask_0, squeeze_mask = var_2608_squeeze_mask_0, x = g_13_cast_fp16)[name = string("op_2608_cast_fp16")];
+            tensor<int32, [2]> var_2609_perm_0 = const()[name = string("op_2609_perm_0"), val = tensor<int32, [2]>([1, 0])];
+            tensor<int32, [3]> var_2610 = const()[name = string("op_2610"), val = tensor<int32, [3]>([32, 16, 64])];
+            tensor<fp16, [32, 1024]> var_2609_cast_fp16 = transpose(perm = var_2609_perm_0, x = var_2608_cast_fp16)[name = string("transpose_21")];
+            tensor<fp16, [32, 16, 64]> g_15_cast_fp16 = reshape(shape = var_2610, x = var_2609_cast_fp16)[name = string("g_15_cast_fp16")];
+            tensor<int32, [1]> g_col_7_axes_0 = const()[name = string("g_col_7_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 16, 64, 1]> g_col_7_cast_fp16 = expand_dims(axes = g_col_7_axes_0, x = g_15_cast_fp16)[name = string("g_col_7_cast_fp16")];
+            bool var_2613_transpose_x_0 = const()[name = string("op_2613_transpose_x_0"), val = bool(false)];
+            bool var_2613_transpose_y_0 = const()[name = string("op_2613_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 16, 64, 1]> var_2613_cast_fp16 = matmul(transpose_x = var_2613_transpose_x_0, transpose_y = var_2613_transpose_y_0, x = layers_0_linear_attn_lower_incl_to_fp16, y = g_col_7_cast_fp16)[name = string("op_2613_cast_fp16")];
+            tensor<int32, [1]> cum_7_axes_0 = const()[name = string("cum_7_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 16, 64]> cum_7_cast_fp16 = squeeze(axes = cum_7_axes_0, x = var_2613_cast_fp16)[name = string("cum_7_cast_fp16")];
+            bool var_2615_transpose_x_0 = const()[name = string("op_2615_transpose_x_0"), val = bool(false)];
+            bool var_2615_transpose_y_0 = const()[name = string("op_2615_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 16, 64, 1]> var_2615_cast_fp16 = matmul(transpose_x = var_2615_transpose_x_0, transpose_y = var_2615_transpose_y_0, x = layers_0_linear_attn_suffix_sum_t_to_fp16, y = g_col_7_cast_fp16)[name = string("op_2615_cast_fp16")];
+            tensor<int32, [1]> to_end_7_axes_0 = const()[name = string("to_end_7_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 16, 64]> to_end_7_cast_fp16 = squeeze(axes = to_end_7_axes_0, x = var_2615_cast_fp16)[name = string("to_end_7_cast_fp16")];
+            bool var_2617_transpose_x_0 = const()[name = string("op_2617_transpose_x_0"), val = bool(false)];
+            bool var_2617_transpose_y_0 = const()[name = string("op_2617_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 16, 4096, 1]> var_2617_cast_fp16 = matmul(transpose_x = var_2617_transpose_x_0, transpose_y = var_2617_transpose_y_0, x = layers_0_linear_attn_pair_sum_t_to_fp16, y = g_col_7_cast_fp16)[name = string("op_2617_cast_fp16")];
+            tensor<int32, [4]> var_2618 = const()[name = string("op_2618"), val = tensor<int32, [4]>([32, 16, 64, 64])];
+            tensor<fp16, [32, 16, 64, 64]> pair_43_cast_fp16 = reshape(shape = var_2618, x = var_2617_cast_fp16)[name = string("pair_43_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 64]> var_2620_cast_fp16 = exp(x = pair_43_cast_fp16)[name = string("op_2620_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 64]> pair_decay_7_cast_fp16 = mul(x = var_2620_cast_fp16, y = layers_0_linear_attn_lower_incl_to_fp16)[name = string("pair_decay_7_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 128]> k_beta_7_cast_fp16 = mul(x = k_37_cast_fp16, y = beta_15_cast_fp16)[name = string("k_beta_7_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 128]> v_beta_7_cast_fp16 = mul(x = v_19_cast_fp16, y = beta_15_cast_fp16)[name = string("v_beta_7_cast_fp16")];
+            bool var_2625_transpose_x_1 = const()[name = string("op_2625_transpose_x_1"), val = bool(false)];
+            bool var_2625_transpose_y_1 = const()[name = string("op_2625_transpose_y_1"), val = bool(true)];
+            tensor<fp16, [32, 16, 64, 64]> var_2625_cast_fp16 = matmul(transpose_x = var_2625_transpose_x_1, transpose_y = var_2625_transpose_y_1, x = k_beta_7_cast_fp16, y = k_37_cast_fp16)[name = string("op_2625_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 64]> kkt_7_cast_fp16 = mul(x = var_2625_cast_fp16, y = pair_decay_7_cast_fp16)[name = string("kkt_7_cast_fp16")];
+            bool var_2628_transpose_x_1 = const()[name = string("op_2628_transpose_x_1"), val = bool(false)];
+            bool var_2628_transpose_y_1 = const()[name = string("op_2628_transpose_y_1"), val = bool(true)];
+            tensor<fp16, [32, 16, 64, 64]> var_2628_cast_fp16 = matmul(transpose_x = var_2628_transpose_x_1, transpose_y = var_2628_transpose_y_1, x = q_43_cast_fp16, y = k_37_cast_fp16)[name = string("op_2628_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 64]> attn_intra_7_cast_fp16 = mul(x = var_2628_cast_fp16, y = pair_decay_7_cast_fp16)[name = string("attn_intra_7_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 64]> var_2630_cast_fp16 = mul(x = kkt_7_cast_fp16, y = layers_0_linear_attn_strict_lower_to_fp16)[name = string("op_2630_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 64]> t_13_cast_fp16 = add(x = layers_0_linear_attn_eye_to_fp16, y = var_2630_cast_fp16)[name = string("t_13_cast_fp16")];
+            tensor<int32, [3]> var_2634 = const()[name = string("op_2634"), val = tensor<int32, [3]>([-1, 64, 64])];
+            tensor<fp16, [512, 64, 64]> t_15_cast_fp16 = reshape(shape = var_2634, x = t_13_cast_fp16)[name = string("t_15_cast_fp16")];
+            tensor<int32, [5]> var_2639 = const()[name = string("op_2639"), val = tensor<int32, [5]>([512, 32, 2, 32, 2])];
+            tensor<fp16, [512, 32, 2, 32, 2]> var_2640_cast_fp16 = reshape(shape = var_2639, x = t_15_cast_fp16)[name = string("op_2640_cast_fp16")];
+            tensor<fp16, [512, 32, 2, 32, 2]> var_2645_cast_fp16 = mul(x = var_2640_cast_fp16, y = var_213_to_fp16)[name = string("op_2645_cast_fp16")];
+            tensor<int32, [1]> blocks_37_axes_0 = const()[name = string("blocks_37_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_37_keep_dims_0 = const()[name = string("blocks_37_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 32, 2, 2]> blocks_37_cast_fp16 = reduce_sum(axes = blocks_37_axes_0, keep_dims = blocks_37_keep_dims_0, x = var_2645_cast_fp16)[name = string("blocks_37_cast_fp16")];
+            tensor<int32, [4]> var_2648_begin_0 = const()[name = string("op_2648_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
+            tensor<int32, [4]> var_2648_end_0 = const()[name = string("op_2648_end_0"), val = tensor<int32, [4]>([512, 32, 2, 2])];
+            tensor<bool, [4]> var_2648_end_mask_0 = const()[name = string("op_2648_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 32, 1, 2]> var_2648_cast_fp16 = slice_by_index(begin = var_2648_begin_0, end = var_2648_end_0, end_mask = var_2648_end_mask_0, x = blocks_37_cast_fp16)[name = string("op_2648_cast_fp16")];
+            tensor<int32, [4]> x_105_begin_0 = const()[name = string("x_105_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_105_end_0 = const()[name = string("x_105_end_0"), val = tensor<int32, [4]>([512, 32, 1, 1])];
+            tensor<bool, [4]> x_105_end_mask_0 = const()[name = string("x_105_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 32, 1, 1]> x_105_cast_fp16 = slice_by_index(begin = x_105_begin_0, end = x_105_end_0, end_mask = x_105_end_mask_0, x = var_2648_cast_fp16)[name = string("x_105_cast_fp16")];
+            bool var_2658_transpose_x_0 = const()[name = string("op_2658_transpose_x_0"), val = bool(false)];
+            bool var_2658_transpose_y_0 = const()[name = string("op_2658_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 32, 1, 1]> var_2658_cast_fp16 = matmul(transpose_x = var_2658_transpose_x_0, transpose_y = var_2658_transpose_y_0, x = x_105_cast_fp16, y = var_221_to_fp16)[name = string("op_2658_cast_fp16")];
+            bool var_2659_transpose_x_0 = const()[name = string("op_2659_transpose_x_0"), val = bool(false)];
+            bool var_2659_transpose_y_0 = const()[name = string("op_2659_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 32, 1, 1]> var_2659_cast_fp16 = matmul(transpose_x = var_2659_transpose_x_0, transpose_y = var_2659_transpose_y_0, x = var_221_to_fp16, y = var_2658_cast_fp16)[name = string("op_2659_cast_fp16")];
+            fp16 const_47_promoted_to_fp16 = const()[name = string("const_47_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 32, 1, 1]> lower_37_cast_fp16 = mul(x = var_2659_cast_fp16, y = const_47_promoted_to_fp16)[name = string("lower_37_cast_fp16")];
+            bool bottom_37_interleave_0 = const()[name = string("bottom_37_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 32, 1, 2]> bottom_37_cast_fp16 = concat(axis = var_2480, interleave = bottom_37_interleave_0, values = (lower_37_cast_fp16, var_221_to_fp16))[name = string("bottom_37_cast_fp16")];
+            bool inv_51_interleave_0 = const()[name = string("inv_51_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 32, 2, 2]> inv_51_cast_fp16 = concat(axis = var_2465, interleave = inv_51_interleave_0, values = (top_1_to_fp16, bottom_37_cast_fp16))[name = string("inv_51_cast_fp16")];
+            tensor<int32, [5]> var_2668 = const()[name = string("op_2668"), val = tensor<int32, [5]>([512, 16, 4, 16, 4])];
+            tensor<fp16, [512, 16, 4, 16, 4]> var_2669_cast_fp16 = reshape(shape = var_2668, x = t_15_cast_fp16)[name = string("op_2669_cast_fp16")];
+            tensor<fp16, [512, 16, 4, 16, 4]> var_2674_cast_fp16 = mul(x = var_2669_cast_fp16, y = var_242_to_fp16)[name = string("op_2674_cast_fp16")];
+            tensor<int32, [1]> blocks_39_axes_0 = const()[name = string("blocks_39_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_39_keep_dims_0 = const()[name = string("blocks_39_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 4, 4]> blocks_39_cast_fp16 = reduce_sum(axes = blocks_39_axes_0, keep_dims = blocks_39_keep_dims_0, x = var_2674_cast_fp16)[name = string("blocks_39_cast_fp16")];
+            tensor<int32, [4]> var_2677_begin_0 = const()[name = string("op_2677_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
+            tensor<int32, [4]> var_2677_end_0 = const()[name = string("op_2677_end_0"), val = tensor<int32, [4]>([512, 16, 4, 4])];
+            tensor<bool, [4]> var_2677_end_mask_0 = const()[name = string("op_2677_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 16, 2, 4]> var_2677_cast_fp16 = slice_by_index(begin = var_2677_begin_0, end = var_2677_end_0, end_mask = var_2677_end_mask_0, x = blocks_39_cast_fp16)[name = string("op_2677_cast_fp16")];
+            tensor<int32, [4]> x_107_begin_0 = const()[name = string("x_107_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_107_end_0 = const()[name = string("x_107_end_0"), val = tensor<int32, [4]>([512, 16, 2, 2])];
+            tensor<bool, [4]> x_107_end_mask_0 = const()[name = string("x_107_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 16, 2, 2]> x_107_cast_fp16 = slice_by_index(begin = x_107_begin_0, end = x_107_end_0, end_mask = x_107_end_mask_0, x = var_2677_cast_fp16)[name = string("x_107_cast_fp16")];
+            tensor<int32, [5]> var_2679 = const()[name = string("op_2679"), val = tensor<int32, [5]>([512, 16, 2, 2, 2])];
+            tensor<fp16, [512, 16, 2, 2, 2]> pair_47_cast_fp16 = reshape(shape = var_2679, x = inv_51_cast_fp16)[name = string("pair_47_cast_fp16")];
+            tensor<int32, [5]> var_2681_begin_0 = const()[name = string("op_2681_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_2681_end_0 = const()[name = string("op_2681_end_0"), val = tensor<int32, [5]>([512, 16, 1, 2, 2])];
+            tensor<bool, [5]> var_2681_end_mask_0 = const()[name = string("op_2681_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_2681_squeeze_mask_0 = const()[name = string("op_2681_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 16, 2, 2]> var_2681_cast_fp16 = slice_by_index(begin = var_2681_begin_0, end = var_2681_end_0, end_mask = var_2681_end_mask_0, squeeze_mask = var_2681_squeeze_mask_0, x = pair_47_cast_fp16)[name = string("op_2681_cast_fp16")];
+            tensor<int32, [5]> var_2684_begin_0 = const()[name = string("op_2684_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_2684_end_0 = const()[name = string("op_2684_end_0"), val = tensor<int32, [5]>([512, 16, 2, 2, 2])];
+            tensor<bool, [5]> var_2684_end_mask_0 = const()[name = string("op_2684_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_2684_squeeze_mask_0 = const()[name = string("op_2684_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 16, 2, 2]> var_2684_cast_fp16 = slice_by_index(begin = var_2684_begin_0, end = var_2684_end_0, end_mask = var_2684_end_mask_0, squeeze_mask = var_2684_squeeze_mask_0, x = pair_47_cast_fp16)[name = string("op_2684_cast_fp16")];
+            bool var_2687_transpose_x_0 = const()[name = string("op_2687_transpose_x_0"), val = bool(false)];
+            bool var_2687_transpose_y_0 = const()[name = string("op_2687_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 2, 2]> var_2687_cast_fp16 = matmul(transpose_x = var_2687_transpose_x_0, transpose_y = var_2687_transpose_y_0, x = x_107_cast_fp16, y = var_2681_cast_fp16)[name = string("op_2687_cast_fp16")];
+            bool var_2688_transpose_x_0 = const()[name = string("op_2688_transpose_x_0"), val = bool(false)];
+            bool var_2688_transpose_y_0 = const()[name = string("op_2688_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 2, 2]> var_2688_cast_fp16 = matmul(transpose_x = var_2688_transpose_x_0, transpose_y = var_2688_transpose_y_0, x = var_2684_cast_fp16, y = var_2687_cast_fp16)[name = string("op_2688_cast_fp16")];
+            fp16 const_48_promoted_to_fp16 = const()[name = string("const_48_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 16, 2, 2]> lower_39_cast_fp16 = mul(x = var_2688_cast_fp16, y = const_48_promoted_to_fp16)[name = string("lower_39_cast_fp16")];
+            bool top_39_interleave_0 = const()[name = string("top_39_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 2, 4]> top_39_cast_fp16 = concat(axis = var_2480, interleave = top_39_interleave_0, values = (var_2681_cast_fp16, var_259_to_fp16))[name = string("top_39_cast_fp16")];
+            bool bottom_39_interleave_0 = const()[name = string("bottom_39_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 2, 4]> bottom_39_cast_fp16 = concat(axis = var_2480, interleave = bottom_39_interleave_0, values = (lower_39_cast_fp16, var_2684_cast_fp16))[name = string("bottom_39_cast_fp16")];
+            bool inv_53_interleave_0 = const()[name = string("inv_53_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 4, 4]> inv_53_cast_fp16 = concat(axis = var_2465, interleave = inv_53_interleave_0, values = (top_39_cast_fp16, bottom_39_cast_fp16))[name = string("inv_53_cast_fp16")];
+            tensor<int32, [5]> var_2697 = const()[name = string("op_2697"), val = tensor<int32, [5]>([512, 8, 8, 8, 8])];
+            tensor<fp16, [512, 8, 8, 8, 8]> var_2698_cast_fp16 = reshape(shape = var_2697, x = t_15_cast_fp16)[name = string("op_2698_cast_fp16")];
+            tensor<fp16, [8, 1, 8, 1]> var_2702_to_fp16 = const()[name = string("op_2702_to_fp16"), val = tensor<fp16, [8, 1, 8, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(687867456)))];
+            tensor<fp16, [512, 8, 8, 8, 8]> var_2703_cast_fp16 = mul(x = var_2698_cast_fp16, y = var_2702_to_fp16)[name = string("op_2703_cast_fp16")];
+            tensor<int32, [1]> blocks_41_axes_0 = const()[name = string("blocks_41_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_41_keep_dims_0 = const()[name = string("blocks_41_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 8, 8]> blocks_41_cast_fp16 = reduce_sum(axes = blocks_41_axes_0, keep_dims = blocks_41_keep_dims_0, x = var_2703_cast_fp16)[name = string("blocks_41_cast_fp16")];
+            tensor<int32, [4]> var_2706_begin_0 = const()[name = string("op_2706_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
+            tensor<int32, [4]> var_2706_end_0 = const()[name = string("op_2706_end_0"), val = tensor<int32, [4]>([512, 8, 8, 8])];
+            tensor<bool, [4]> var_2706_end_mask_0 = const()[name = string("op_2706_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 8, 4, 8]> var_2706_cast_fp16 = slice_by_index(begin = var_2706_begin_0, end = var_2706_end_0, end_mask = var_2706_end_mask_0, x = blocks_41_cast_fp16)[name = string("op_2706_cast_fp16")];
+            tensor<int32, [4]> x_109_begin_0 = const()[name = string("x_109_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_109_end_0 = const()[name = string("x_109_end_0"), val = tensor<int32, [4]>([512, 8, 4, 4])];
+            tensor<bool, [4]> x_109_end_mask_0 = const()[name = string("x_109_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 8, 4, 4]> x_109_cast_fp16 = slice_by_index(begin = x_109_begin_0, end = x_109_end_0, end_mask = x_109_end_mask_0, x = var_2706_cast_fp16)[name = string("x_109_cast_fp16")];
+            tensor<int32, [5]> var_2708 = const()[name = string("op_2708"), val = tensor<int32, [5]>([512, 8, 2, 4, 4])];
+            tensor<fp16, [512, 8, 2, 4, 4]> pair_49_cast_fp16 = reshape(shape = var_2708, x = inv_53_cast_fp16)[name = string("pair_49_cast_fp16")];
+            tensor<int32, [5]> var_2710_begin_0 = const()[name = string("op_2710_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_2710_end_0 = const()[name = string("op_2710_end_0"), val = tensor<int32, [5]>([512, 8, 1, 4, 4])];
+            tensor<bool, [5]> var_2710_end_mask_0 = const()[name = string("op_2710_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_2710_squeeze_mask_0 = const()[name = string("op_2710_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 8, 4, 4]> var_2710_cast_fp16 = slice_by_index(begin = var_2710_begin_0, end = var_2710_end_0, end_mask = var_2710_end_mask_0, squeeze_mask = var_2710_squeeze_mask_0, x = pair_49_cast_fp16)[name = string("op_2710_cast_fp16")];
+            tensor<int32, [5]> var_2713_begin_0 = const()[name = string("op_2713_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_2713_end_0 = const()[name = string("op_2713_end_0"), val = tensor<int32, [5]>([512, 8, 2, 4, 4])];
+            tensor<bool, [5]> var_2713_end_mask_0 = const()[name = string("op_2713_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_2713_squeeze_mask_0 = const()[name = string("op_2713_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 8, 4, 4]> var_2713_cast_fp16 = slice_by_index(begin = var_2713_begin_0, end = var_2713_end_0, end_mask = var_2713_end_mask_0, squeeze_mask = var_2713_squeeze_mask_0, x = pair_49_cast_fp16)[name = string("op_2713_cast_fp16")];
+            bool var_2716_transpose_x_0 = const()[name = string("op_2716_transpose_x_0"), val = bool(false)];
+            bool var_2716_transpose_y_0 = const()[name = string("op_2716_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 4, 4]> var_2716_cast_fp16 = matmul(transpose_x = var_2716_transpose_x_0, transpose_y = var_2716_transpose_y_0, x = x_109_cast_fp16, y = var_2710_cast_fp16)[name = string("op_2716_cast_fp16")];
+            bool var_2717_transpose_x_0 = const()[name = string("op_2717_transpose_x_0"), val = bool(false)];
+            bool var_2717_transpose_y_0 = const()[name = string("op_2717_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 4, 4]> var_2717_cast_fp16 = matmul(transpose_x = var_2717_transpose_x_0, transpose_y = var_2717_transpose_y_0, x = var_2713_cast_fp16, y = var_2716_cast_fp16)[name = string("op_2717_cast_fp16")];
+            fp16 const_49_promoted_to_fp16 = const()[name = string("const_49_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 8, 4, 4]> lower_41_cast_fp16 = mul(x = var_2717_cast_fp16, y = const_49_promoted_to_fp16)[name = string("lower_41_cast_fp16")];
+            bool top_41_interleave_0 = const()[name = string("top_41_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 4, 8]> top_41_cast_fp16 = concat(axis = var_2480, interleave = top_41_interleave_0, values = (var_2710_cast_fp16, var_288_to_fp16))[name = string("top_41_cast_fp16")];
+            bool bottom_41_interleave_0 = const()[name = string("bottom_41_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 4, 8]> bottom_41_cast_fp16 = concat(axis = var_2480, interleave = bottom_41_interleave_0, values = (lower_41_cast_fp16, var_2713_cast_fp16))[name = string("bottom_41_cast_fp16")];
+            bool inv_55_interleave_0 = const()[name = string("inv_55_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 8, 8]> inv_55_cast_fp16 = concat(axis = var_2465, interleave = inv_55_interleave_0, values = (top_41_cast_fp16, bottom_41_cast_fp16))[name = string("inv_55_cast_fp16")];
+            tensor<int32, [5]> var_2726 = const()[name = string("op_2726"), val = tensor<int32, [5]>([512, 4, 16, 4, 16])];
+            tensor<fp16, [512, 4, 16, 4, 16]> var_2727_cast_fp16 = reshape(shape = var_2726, x = t_15_cast_fp16)[name = string("op_2727_cast_fp16")];
+            tensor<fp16, [4, 1, 4, 1]> var_2731_to_fp16 = const()[name = string("op_2731_to_fp16"), val = tensor<fp16, [4, 1, 4, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(687867648)))];
+            tensor<fp16, [512, 4, 16, 4, 16]> var_2732_cast_fp16 = mul(x = var_2727_cast_fp16, y = var_2731_to_fp16)[name = string("op_2732_cast_fp16")];
+            tensor<int32, [1]> blocks_43_axes_0 = const()[name = string("blocks_43_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_43_keep_dims_0 = const()[name = string("blocks_43_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 16, 16]> blocks_43_cast_fp16 = reduce_sum(axes = blocks_43_axes_0, keep_dims = blocks_43_keep_dims_0, x = var_2732_cast_fp16)[name = string("blocks_43_cast_fp16")];
+            tensor<int32, [4]> var_2735_begin_0 = const()[name = string("op_2735_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
+            tensor<int32, [4]> var_2735_end_0 = const()[name = string("op_2735_end_0"), val = tensor<int32, [4]>([512, 4, 16, 16])];
+            tensor<bool, [4]> var_2735_end_mask_0 = const()[name = string("op_2735_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 4, 8, 16]> var_2735_cast_fp16 = slice_by_index(begin = var_2735_begin_0, end = var_2735_end_0, end_mask = var_2735_end_mask_0, x = blocks_43_cast_fp16)[name = string("op_2735_cast_fp16")];
+            tensor<int32, [4]> x_111_begin_0 = const()[name = string("x_111_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_111_end_0 = const()[name = string("x_111_end_0"), val = tensor<int32, [4]>([512, 4, 8, 8])];
+            tensor<bool, [4]> x_111_end_mask_0 = const()[name = string("x_111_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 4, 8, 8]> x_111_cast_fp16 = slice_by_index(begin = x_111_begin_0, end = x_111_end_0, end_mask = x_111_end_mask_0, x = var_2735_cast_fp16)[name = string("x_111_cast_fp16")];
+            tensor<int32, [5]> var_2737 = const()[name = string("op_2737"), val = tensor<int32, [5]>([512, 4, 2, 8, 8])];
+            tensor<fp16, [512, 4, 2, 8, 8]> pair_51_cast_fp16 = reshape(shape = var_2737, x = inv_55_cast_fp16)[name = string("pair_51_cast_fp16")];
+            tensor<int32, [5]> var_2739_begin_0 = const()[name = string("op_2739_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_2739_end_0 = const()[name = string("op_2739_end_0"), val = tensor<int32, [5]>([512, 4, 1, 8, 8])];
+            tensor<bool, [5]> var_2739_end_mask_0 = const()[name = string("op_2739_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_2739_squeeze_mask_0 = const()[name = string("op_2739_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 4, 8, 8]> var_2739_cast_fp16 = slice_by_index(begin = var_2739_begin_0, end = var_2739_end_0, end_mask = var_2739_end_mask_0, squeeze_mask = var_2739_squeeze_mask_0, x = pair_51_cast_fp16)[name = string("op_2739_cast_fp16")];
+            tensor<int32, [5]> var_2742_begin_0 = const()[name = string("op_2742_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_2742_end_0 = const()[name = string("op_2742_end_0"), val = tensor<int32, [5]>([512, 4, 2, 8, 8])];
+            tensor<bool, [5]> var_2742_end_mask_0 = const()[name = string("op_2742_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_2742_squeeze_mask_0 = const()[name = string("op_2742_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 4, 8, 8]> var_2742_cast_fp16 = slice_by_index(begin = var_2742_begin_0, end = var_2742_end_0, end_mask = var_2742_end_mask_0, squeeze_mask = var_2742_squeeze_mask_0, x = pair_51_cast_fp16)[name = string("op_2742_cast_fp16")];
+            bool var_2745_transpose_x_0 = const()[name = string("op_2745_transpose_x_0"), val = bool(false)];
+            bool var_2745_transpose_y_0 = const()[name = string("op_2745_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 8, 8]> var_2745_cast_fp16 = matmul(transpose_x = var_2745_transpose_x_0, transpose_y = var_2745_transpose_y_0, x = x_111_cast_fp16, y = var_2739_cast_fp16)[name = string("op_2745_cast_fp16")];
+            bool var_2746_transpose_x_0 = const()[name = string("op_2746_transpose_x_0"), val = bool(false)];
+            bool var_2746_transpose_y_0 = const()[name = string("op_2746_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 8, 8]> var_2746_cast_fp16 = matmul(transpose_x = var_2746_transpose_x_0, transpose_y = var_2746_transpose_y_0, x = var_2742_cast_fp16, y = var_2745_cast_fp16)[name = string("op_2746_cast_fp16")];
+            fp16 const_50_promoted_to_fp16 = const()[name = string("const_50_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 4, 8, 8]> lower_43_cast_fp16 = mul(x = var_2746_cast_fp16, y = const_50_promoted_to_fp16)[name = string("lower_43_cast_fp16")];
+            bool top_43_interleave_0 = const()[name = string("top_43_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 8, 16]> top_43_cast_fp16 = concat(axis = var_2480, interleave = top_43_interleave_0, values = (var_2739_cast_fp16, var_317_to_fp16))[name = string("top_43_cast_fp16")];
+            bool bottom_43_interleave_0 = const()[name = string("bottom_43_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 8, 16]> bottom_43_cast_fp16 = concat(axis = var_2480, interleave = bottom_43_interleave_0, values = (lower_43_cast_fp16, var_2742_cast_fp16))[name = string("bottom_43_cast_fp16")];
+            bool inv_57_interleave_0 = const()[name = string("inv_57_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 16, 16]> inv_57_cast_fp16 = concat(axis = var_2465, interleave = inv_57_interleave_0, values = (top_43_cast_fp16, bottom_43_cast_fp16))[name = string("inv_57_cast_fp16")];
+            tensor<int32, [5]> var_2755 = const()[name = string("op_2755"), val = tensor<int32, [5]>([512, 2, 32, 2, 32])];
+            tensor<fp16, [512, 2, 32, 2, 32]> var_2756_cast_fp16 = reshape(shape = var_2755, x = t_15_cast_fp16)[name = string("op_2756_cast_fp16")];
+            tensor<fp16, [2, 1, 2, 1]> var_2760_to_fp16 = const()[name = string("op_2760_to_fp16"), val = tensor<fp16, [2, 1, 2, 1]>([[[[0x1p+0], [0x0p+0]]], [[[0x0p+0], [0x1p+0]]]])];
+            tensor<fp16, [512, 2, 32, 2, 32]> var_2761_cast_fp16 = mul(x = var_2756_cast_fp16, y = var_2760_to_fp16)[name = string("op_2761_cast_fp16")];
+            tensor<int32, [1]> blocks_45_axes_0 = const()[name = string("blocks_45_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_45_keep_dims_0 = const()[name = string("blocks_45_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 32, 32]> blocks_45_cast_fp16 = reduce_sum(axes = blocks_45_axes_0, keep_dims = blocks_45_keep_dims_0, x = var_2761_cast_fp16)[name = string("blocks_45_cast_fp16")];
+            tensor<int32, [4]> var_2764_begin_0 = const()[name = string("op_2764_begin_0"), val = tensor<int32, [4]>([0, 0, 16, 0])];
+            tensor<int32, [4]> var_2764_end_0 = const()[name = string("op_2764_end_0"), val = tensor<int32, [4]>([512, 2, 32, 32])];
+            tensor<bool, [4]> var_2764_end_mask_0 = const()[name = string("op_2764_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 2, 16, 32]> var_2764_cast_fp16 = slice_by_index(begin = var_2764_begin_0, end = var_2764_end_0, end_mask = var_2764_end_mask_0, x = blocks_45_cast_fp16)[name = string("op_2764_cast_fp16")];
+            tensor<int32, [4]> x_113_begin_0 = const()[name = string("x_113_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_113_end_0 = const()[name = string("x_113_end_0"), val = tensor<int32, [4]>([512, 2, 16, 16])];
+            tensor<bool, [4]> x_113_end_mask_0 = const()[name = string("x_113_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 2, 16, 16]> x_113_cast_fp16 = slice_by_index(begin = x_113_begin_0, end = x_113_end_0, end_mask = x_113_end_mask_0, x = var_2764_cast_fp16)[name = string("x_113_cast_fp16")];
+            tensor<int32, [5]> var_2766 = const()[name = string("op_2766"), val = tensor<int32, [5]>([512, 2, 2, 16, 16])];
+            tensor<fp16, [512, 2, 2, 16, 16]> pair_53_cast_fp16 = reshape(shape = var_2766, x = inv_57_cast_fp16)[name = string("pair_53_cast_fp16")];
+            tensor<int32, [5]> var_2768_begin_0 = const()[name = string("op_2768_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_2768_end_0 = const()[name = string("op_2768_end_0"), val = tensor<int32, [5]>([512, 2, 1, 16, 16])];
+            tensor<bool, [5]> var_2768_end_mask_0 = const()[name = string("op_2768_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_2768_squeeze_mask_0 = const()[name = string("op_2768_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 2, 16, 16]> var_2768_cast_fp16 = slice_by_index(begin = var_2768_begin_0, end = var_2768_end_0, end_mask = var_2768_end_mask_0, squeeze_mask = var_2768_squeeze_mask_0, x = pair_53_cast_fp16)[name = string("op_2768_cast_fp16")];
+            tensor<int32, [5]> var_2771_begin_0 = const()[name = string("op_2771_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_2771_end_0 = const()[name = string("op_2771_end_0"), val = tensor<int32, [5]>([512, 2, 2, 16, 16])];
+            tensor<bool, [5]> var_2771_end_mask_0 = const()[name = string("op_2771_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_2771_squeeze_mask_0 = const()[name = string("op_2771_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 2, 16, 16]> var_2771_cast_fp16 = slice_by_index(begin = var_2771_begin_0, end = var_2771_end_0, end_mask = var_2771_end_mask_0, squeeze_mask = var_2771_squeeze_mask_0, x = pair_53_cast_fp16)[name = string("op_2771_cast_fp16")];
+            bool var_2774_transpose_x_0 = const()[name = string("op_2774_transpose_x_0"), val = bool(false)];
+            bool var_2774_transpose_y_0 = const()[name = string("op_2774_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 16, 16]> var_2774_cast_fp16 = matmul(transpose_x = var_2774_transpose_x_0, transpose_y = var_2774_transpose_y_0, x = x_113_cast_fp16, y = var_2768_cast_fp16)[name = string("op_2774_cast_fp16")];
+            bool var_2775_transpose_x_0 = const()[name = string("op_2775_transpose_x_0"), val = bool(false)];
+            bool var_2775_transpose_y_0 = const()[name = string("op_2775_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 16, 16]> var_2775_cast_fp16 = matmul(transpose_x = var_2775_transpose_x_0, transpose_y = var_2775_transpose_y_0, x = var_2771_cast_fp16, y = var_2774_cast_fp16)[name = string("op_2775_cast_fp16")];
+            fp16 const_51_promoted_to_fp16 = const()[name = string("const_51_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 2, 16, 16]> lower_45_cast_fp16 = mul(x = var_2775_cast_fp16, y = const_51_promoted_to_fp16)[name = string("lower_45_cast_fp16")];
+            bool top_45_interleave_0 = const()[name = string("top_45_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 16, 32]> top_45_cast_fp16 = concat(axis = var_2480, interleave = top_45_interleave_0, values = (var_2768_cast_fp16, var_346_to_fp16))[name = string("top_45_cast_fp16")];
+            bool bottom_45_interleave_0 = const()[name = string("bottom_45_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 16, 32]> bottom_45_cast_fp16 = concat(axis = var_2480, interleave = bottom_45_interleave_0, values = (lower_45_cast_fp16, var_2771_cast_fp16))[name = string("bottom_45_cast_fp16")];
+            bool inv_59_interleave_0 = const()[name = string("inv_59_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 32, 32]> inv_59_cast_fp16 = concat(axis = var_2465, interleave = inv_59_interleave_0, values = (top_45_cast_fp16, bottom_45_cast_fp16))[name = string("inv_59_cast_fp16")];
+            tensor<int32, [5]> var_2784 = const()[name = string("op_2784"), val = tensor<int32, [5]>([512, 1, 64, 1, 64])];
+            tensor<fp16, [512, 1, 64, 1, 64]> var_2785_cast_fp16 = reshape(shape = var_2784, x = t_15_cast_fp16)[name = string("op_2785_cast_fp16")];
+            tensor<int32, [1]> blocks_47_axes_0 = const()[name = string("blocks_47_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_47_keep_dims_0 = const()[name = string("blocks_47_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 64, 64]> blocks_47_cast_fp16 = reduce_sum(axes = blocks_47_axes_0, keep_dims = blocks_47_keep_dims_0, x = var_2785_cast_fp16)[name = string("blocks_47_cast_fp16")];
+            tensor<int32, [4]> var_2793_begin_0 = const()[name = string("op_2793_begin_0"), val = tensor<int32, [4]>([0, 0, 32, 0])];
+            tensor<int32, [4]> var_2793_end_0 = const()[name = string("op_2793_end_0"), val = tensor<int32, [4]>([512, 1, 64, 64])];
+            tensor<bool, [4]> var_2793_end_mask_0 = const()[name = string("op_2793_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 1, 32, 64]> var_2793_cast_fp16 = slice_by_index(begin = var_2793_begin_0, end = var_2793_end_0, end_mask = var_2793_end_mask_0, x = blocks_47_cast_fp16)[name = string("op_2793_cast_fp16")];
+            tensor<int32, [4]> x_115_begin_0 = const()[name = string("x_115_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_115_end_0 = const()[name = string("x_115_end_0"), val = tensor<int32, [4]>([512, 1, 32, 32])];
+            tensor<bool, [4]> x_115_end_mask_0 = const()[name = string("x_115_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 1, 32, 32]> x_115_cast_fp16 = slice_by_index(begin = x_115_begin_0, end = x_115_end_0, end_mask = x_115_end_mask_0, x = var_2793_cast_fp16)[name = string("x_115_cast_fp16")];
+            tensor<int32, [5]> var_2795 = const()[name = string("op_2795"), val = tensor<int32, [5]>([512, 1, 2, 32, 32])];
+            tensor<fp16, [512, 1, 2, 32, 32]> pair_55_cast_fp16 = reshape(shape = var_2795, x = inv_59_cast_fp16)[name = string("pair_55_cast_fp16")];
+            tensor<int32, [5]> var_2797_begin_0 = const()[name = string("op_2797_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_2797_end_0 = const()[name = string("op_2797_end_0"), val = tensor<int32, [5]>([512, 1, 1, 32, 32])];
+            tensor<bool, [5]> var_2797_end_mask_0 = const()[name = string("op_2797_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_2797_squeeze_mask_0 = const()[name = string("op_2797_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 1, 32, 32]> var_2797_cast_fp16 = slice_by_index(begin = var_2797_begin_0, end = var_2797_end_0, end_mask = var_2797_end_mask_0, squeeze_mask = var_2797_squeeze_mask_0, x = pair_55_cast_fp16)[name = string("op_2797_cast_fp16")];
+            tensor<int32, [5]> var_2800_begin_0 = const()[name = string("op_2800_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_2800_end_0 = const()[name = string("op_2800_end_0"), val = tensor<int32, [5]>([512, 1, 2, 32, 32])];
+            tensor<bool, [5]> var_2800_end_mask_0 = const()[name = string("op_2800_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_2800_squeeze_mask_0 = const()[name = string("op_2800_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 1, 32, 32]> var_2800_cast_fp16 = slice_by_index(begin = var_2800_begin_0, end = var_2800_end_0, end_mask = var_2800_end_mask_0, squeeze_mask = var_2800_squeeze_mask_0, x = pair_55_cast_fp16)[name = string("op_2800_cast_fp16")];
+            bool var_2803_transpose_x_0 = const()[name = string("op_2803_transpose_x_0"), val = bool(false)];
+            bool var_2803_transpose_y_0 = const()[name = string("op_2803_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 32, 32]> var_2803_cast_fp16 = matmul(transpose_x = var_2803_transpose_x_0, transpose_y = var_2803_transpose_y_0, x = x_115_cast_fp16, y = var_2797_cast_fp16)[name = string("op_2803_cast_fp16")];
+            bool var_2804_transpose_x_0 = const()[name = string("op_2804_transpose_x_0"), val = bool(false)];
+            bool var_2804_transpose_y_0 = const()[name = string("op_2804_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 32, 32]> var_2804_cast_fp16 = matmul(transpose_x = var_2804_transpose_x_0, transpose_y = var_2804_transpose_y_0, x = var_2800_cast_fp16, y = var_2803_cast_fp16)[name = string("op_2804_cast_fp16")];
+            fp16 const_52_promoted_to_fp16 = const()[name = string("const_52_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 1, 32, 32]> lower_47_cast_fp16 = mul(x = var_2804_cast_fp16, y = const_52_promoted_to_fp16)[name = string("lower_47_cast_fp16")];
+            bool top_47_interleave_0 = const()[name = string("top_47_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 32, 64]> top_47_cast_fp16 = concat(axis = var_2480, interleave = top_47_interleave_0, values = (var_2797_cast_fp16, var_375_to_fp16))[name = string("top_47_cast_fp16")];
+            bool bottom_47_interleave_0 = const()[name = string("bottom_47_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 32, 64]> bottom_47_cast_fp16 = concat(axis = var_2480, interleave = bottom_47_interleave_0, values = (lower_47_cast_fp16, var_2800_cast_fp16))[name = string("bottom_47_cast_fp16")];
+            bool inv_61_interleave_0 = const()[name = string("inv_61_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 64, 64]> inv_61_cast_fp16 = concat(axis = var_2465, interleave = inv_61_interleave_0, values = (top_47_cast_fp16, bottom_47_cast_fp16))[name = string("inv_61_cast_fp16")];
+            tensor<int32, [4]> var_2813 = const()[name = string("op_2813"), val = tensor<int32, [4]>([32, 16, 64, 64])];
+            tensor<fp16, [32, 16, 64, 64]> inv_63_cast_fp16 = reshape(shape = var_2813, x = inv_61_cast_fp16)[name = string("inv_63_cast_fp16")];
+            bool new_v_7_transpose_x_0 = const()[name = string("new_v_7_transpose_x_0"), val = bool(false)];
+            bool new_v_7_transpose_y_0 = const()[name = string("new_v_7_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 16, 64, 128]> new_v_7_cast_fp16 = matmul(transpose_x = new_v_7_transpose_x_0, transpose_y = new_v_7_transpose_y_0, x = inv_63_cast_fp16, y = v_beta_7_cast_fp16)[name = string("new_v_7_cast_fp16")];
+            tensor<fp16, [32, 16, 64]> var_2816_cast_fp16 = exp(x = cum_7_cast_fp16)[name = string("op_2816_cast_fp16")];
+            tensor<int32, [1]> var_2817_axes_0 = const()[name = string("op_2817_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 16, 64, 1]> var_2817_cast_fp16 = expand_dims(axes = var_2817_axes_0, x = var_2816_cast_fp16)[name = string("op_2817_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 128]> var_2818_cast_fp16 = mul(x = k_beta_7_cast_fp16, y = var_2817_cast_fp16)[name = string("op_2818_cast_fp16")];
+            bool k_cumdecay_7_transpose_x_0 = const()[name = string("k_cumdecay_7_transpose_x_0"), val = bool(false)];
+            bool k_cumdecay_7_transpose_y_0 = const()[name = string("k_cumdecay_7_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 16, 64, 128]> k_cumdecay_7_cast_fp16 = matmul(transpose_x = k_cumdecay_7_transpose_x_0, transpose_y = k_cumdecay_7_transpose_y_0, x = inv_63_cast_fp16, y = var_2818_cast_fp16)[name = string("k_cumdecay_7_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 128]> q_dec_7_cast_fp16 = mul(x = q_43_cast_fp16, y = var_2817_cast_fp16)[name = string("q_dec_7_cast_fp16")];
+            tensor<fp16, [32, 16, 64]> var_2823_cast_fp16 = exp(x = to_end_7_cast_fp16)[name = string("op_2823_cast_fp16")];
+            tensor<int32, [1]> var_2824_axes_0 = const()[name = string("op_2824_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 16, 64, 1]> var_2824_cast_fp16 = expand_dims(axes = var_2824_axes_0, x = var_2823_cast_fp16)[name = string("op_2824_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 128]> k_dec_7_cast_fp16 = mul(x = k_37_cast_fp16, y = var_2824_cast_fp16)[name = string("k_dec_7_cast_fp16")];
+            tensor<int32, [3]> var_2826_begin_0 = const()[name = string("op_2826_begin_0"), val = tensor<int32, [3]>([0, 0, -1])];
+            tensor<int32, [3]> var_2826_end_0 = const()[name = string("op_2826_end_0"), val = tensor<int32, [3]>([32, 16, 64])];
+            tensor<bool, [3]> var_2826_end_mask_0 = const()[name = string("op_2826_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
+            tensor<fp16, [32, 16, 1]> var_2826_cast_fp16 = slice_by_index(begin = var_2826_begin_0, end = var_2826_end_0, end_mask = var_2826_end_mask_0, x = cum_7_cast_fp16)[name = string("op_2826_cast_fp16")];
+            tensor<fp16, [32, 16, 1]> var_2827_cast_fp16 = exp(x = var_2826_cast_fp16)[name = string("op_2827_cast_fp16")];
+            tensor<int32, [1]> chunk_decay_7_axes_0 = const()[name = string("chunk_decay_7_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 16, 1, 1]> chunk_decay_7_cast_fp16 = expand_dims(axes = chunk_decay_7_axes_0, x = var_2827_cast_fp16)[name = string("chunk_decay_7_cast_fp16")];
+            tensor<int32, [4]> var_2832_begin_0 = const()[name = string("op_2832_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_2832_end_0 = const()[name = string("op_2832_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_2832_end_mask_0 = const()[name = string("op_2832_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2832_squeeze_mask_0 = const()[name = string("op_2832_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2832_cast_fp16 = slice_by_index(begin = var_2832_begin_0, end = var_2832_end_0, end_mask = var_2832_end_mask_0, squeeze_mask = var_2832_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_2832_cast_fp16")];
+            tensor<int32, [4]> var_2834_begin_0 = const()[name = string("op_2834_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_2834_end_0 = const()[name = string("op_2834_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_2834_end_mask_0 = const()[name = string("op_2834_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2834_squeeze_mask_0 = const()[name = string("op_2834_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2834_cast_fp16 = slice_by_index(begin = var_2834_begin_0, end = var_2834_end_0, end_mask = var_2834_end_mask_0, squeeze_mask = var_2834_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_2834_cast_fp16")];
+            bool var_2835_transpose_x_0 = const()[name = string("op_2835_transpose_x_0"), val = bool(false)];
+            bool var_2835_transpose_y_0 = const()[name = string("op_2835_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2835_cast_fp16 = matmul(transpose_x = var_2835_transpose_x_0, transpose_y = var_2835_transpose_y_0, x = var_2834_cast_fp16, y = state_1_to_fp16)[name = string("op_2835_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_97_cast_fp16 = sub(x = var_2832_cast_fp16, y = var_2835_cast_fp16)[name = string("v_new_97_cast_fp16")];
+            tensor<int32, [4]> var_2838_begin_0 = const()[name = string("op_2838_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_2838_end_0 = const()[name = string("op_2838_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_2838_end_mask_0 = const()[name = string("op_2838_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2838_squeeze_mask_0 = const()[name = string("op_2838_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2838_cast_fp16 = slice_by_index(begin = var_2838_begin_0, end = var_2838_end_0, end_mask = var_2838_end_mask_0, squeeze_mask = var_2838_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_2838_cast_fp16")];
+            bool var_2839_transpose_x_0 = const()[name = string("op_2839_transpose_x_0"), val = bool(false)];
+            bool var_2839_transpose_y_0 = const()[name = string("op_2839_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2839_cast_fp16 = matmul(transpose_x = var_2839_transpose_x_0, transpose_y = var_2839_transpose_y_0, x = var_2838_cast_fp16, y = state_1_to_fp16)[name = string("op_2839_cast_fp16")];
+            tensor<int32, [4]> var_2841_begin_0 = const()[name = string("op_2841_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_2841_end_0 = const()[name = string("op_2841_end_0"), val = tensor<int32, [4]>([32, 1, 64, 64])];
+            tensor<bool, [4]> var_2841_end_mask_0 = const()[name = string("op_2841_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2841_squeeze_mask_0 = const()[name = string("op_2841_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2841_cast_fp16 = slice_by_index(begin = var_2841_begin_0, end = var_2841_end_0, end_mask = var_2841_end_mask_0, squeeze_mask = var_2841_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_2841_cast_fp16")];
+            bool var_2842_transpose_x_0 = const()[name = string("op_2842_transpose_x_0"), val = bool(false)];
+            bool var_2842_transpose_y_0 = const()[name = string("op_2842_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2842_cast_fp16 = matmul(transpose_x = var_2842_transpose_x_0, transpose_y = var_2842_transpose_y_0, x = var_2841_cast_fp16, y = v_new_97_cast_fp16)[name = string("op_2842_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2843_cast_fp16 = add(x = var_2839_cast_fp16, y = var_2842_cast_fp16)[name = string("op_2843_cast_fp16")];
+            tensor<int32, [4]> var_2845_begin_0 = const()[name = string("op_2845_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_2845_end_0 = const()[name = string("op_2845_end_0"), val = tensor<int32, [4]>([32, 1, 1, 1])];
+            tensor<bool, [4]> var_2845_end_mask_0 = const()[name = string("op_2845_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2845_squeeze_mask_0 = const()[name = string("op_2845_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2845_cast_fp16 = slice_by_index(begin = var_2845_begin_0, end = var_2845_end_0, end_mask = var_2845_end_mask_0, squeeze_mask = var_2845_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_2845_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2846_cast_fp16 = mul(x = state_1_to_fp16, y = var_2845_cast_fp16)[name = string("op_2846_cast_fp16")];
+            tensor<int32, [4]> var_2848_begin_0 = const()[name = string("op_2848_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_2848_end_0 = const()[name = string("op_2848_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_2848_end_mask_0 = const()[name = string("op_2848_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2848_squeeze_mask_0 = const()[name = string("op_2848_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2848_cast_fp16 = slice_by_index(begin = var_2848_begin_0, end = var_2848_end_0, end_mask = var_2848_end_mask_0, squeeze_mask = var_2848_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_2848_cast_fp16")];
+            bool var_2850_transpose_x_1 = const()[name = string("op_2850_transpose_x_1"), val = bool(true)];
+            bool var_2850_transpose_y_1 = const()[name = string("op_2850_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2850_cast_fp16 = matmul(transpose_x = var_2850_transpose_x_1, transpose_y = var_2850_transpose_y_1, x = var_2848_cast_fp16, y = v_new_97_cast_fp16)[name = string("op_2850_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_99_cast_fp16 = add(x = var_2846_cast_fp16, y = var_2850_cast_fp16)[name = string("state_99_cast_fp16")];
+            tensor<int32, [4]> var_2853_begin_0 = const()[name = string("op_2853_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_2853_end_0 = const()[name = string("op_2853_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_2853_end_mask_0 = const()[name = string("op_2853_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2853_squeeze_mask_0 = const()[name = string("op_2853_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2853_cast_fp16 = slice_by_index(begin = var_2853_begin_0, end = var_2853_end_0, end_mask = var_2853_end_mask_0, squeeze_mask = var_2853_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_2853_cast_fp16")];
+            tensor<int32, [4]> var_2855_begin_0 = const()[name = string("op_2855_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_2855_end_0 = const()[name = string("op_2855_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_2855_end_mask_0 = const()[name = string("op_2855_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2855_squeeze_mask_0 = const()[name = string("op_2855_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2855_cast_fp16 = slice_by_index(begin = var_2855_begin_0, end = var_2855_end_0, end_mask = var_2855_end_mask_0, squeeze_mask = var_2855_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_2855_cast_fp16")];
+            bool var_2856_transpose_x_0 = const()[name = string("op_2856_transpose_x_0"), val = bool(false)];
+            bool var_2856_transpose_y_0 = const()[name = string("op_2856_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2856_cast_fp16 = matmul(transpose_x = var_2856_transpose_x_0, transpose_y = var_2856_transpose_y_0, x = var_2855_cast_fp16, y = state_99_cast_fp16)[name = string("op_2856_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_99_cast_fp16 = sub(x = var_2853_cast_fp16, y = var_2856_cast_fp16)[name = string("v_new_99_cast_fp16")];
+            tensor<int32, [4]> var_2859_begin_0 = const()[name = string("op_2859_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_2859_end_0 = const()[name = string("op_2859_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_2859_end_mask_0 = const()[name = string("op_2859_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2859_squeeze_mask_0 = const()[name = string("op_2859_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2859_cast_fp16 = slice_by_index(begin = var_2859_begin_0, end = var_2859_end_0, end_mask = var_2859_end_mask_0, squeeze_mask = var_2859_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_2859_cast_fp16")];
+            bool var_2860_transpose_x_0 = const()[name = string("op_2860_transpose_x_0"), val = bool(false)];
+            bool var_2860_transpose_y_0 = const()[name = string("op_2860_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2860_cast_fp16 = matmul(transpose_x = var_2860_transpose_x_0, transpose_y = var_2860_transpose_y_0, x = var_2859_cast_fp16, y = state_99_cast_fp16)[name = string("op_2860_cast_fp16")];
+            tensor<int32, [4]> var_2862_begin_0 = const()[name = string("op_2862_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_2862_end_0 = const()[name = string("op_2862_end_0"), val = tensor<int32, [4]>([32, 2, 64, 64])];
+            tensor<bool, [4]> var_2862_end_mask_0 = const()[name = string("op_2862_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2862_squeeze_mask_0 = const()[name = string("op_2862_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2862_cast_fp16 = slice_by_index(begin = var_2862_begin_0, end = var_2862_end_0, end_mask = var_2862_end_mask_0, squeeze_mask = var_2862_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_2862_cast_fp16")];
+            bool var_2863_transpose_x_0 = const()[name = string("op_2863_transpose_x_0"), val = bool(false)];
+            bool var_2863_transpose_y_0 = const()[name = string("op_2863_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2863_cast_fp16 = matmul(transpose_x = var_2863_transpose_x_0, transpose_y = var_2863_transpose_y_0, x = var_2862_cast_fp16, y = v_new_99_cast_fp16)[name = string("op_2863_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2864_cast_fp16 = add(x = var_2860_cast_fp16, y = var_2863_cast_fp16)[name = string("op_2864_cast_fp16")];
+            tensor<int32, [4]> var_2866_begin_0 = const()[name = string("op_2866_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_2866_end_0 = const()[name = string("op_2866_end_0"), val = tensor<int32, [4]>([32, 2, 1, 1])];
+            tensor<bool, [4]> var_2866_end_mask_0 = const()[name = string("op_2866_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2866_squeeze_mask_0 = const()[name = string("op_2866_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2866_cast_fp16 = slice_by_index(begin = var_2866_begin_0, end = var_2866_end_0, end_mask = var_2866_end_mask_0, squeeze_mask = var_2866_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_2866_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2867_cast_fp16 = mul(x = state_99_cast_fp16, y = var_2866_cast_fp16)[name = string("op_2867_cast_fp16")];
+            tensor<int32, [4]> var_2869_begin_0 = const()[name = string("op_2869_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_2869_end_0 = const()[name = string("op_2869_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_2869_end_mask_0 = const()[name = string("op_2869_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2869_squeeze_mask_0 = const()[name = string("op_2869_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2869_cast_fp16 = slice_by_index(begin = var_2869_begin_0, end = var_2869_end_0, end_mask = var_2869_end_mask_0, squeeze_mask = var_2869_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_2869_cast_fp16")];
+            bool var_2871_transpose_x_1 = const()[name = string("op_2871_transpose_x_1"), val = bool(true)];
+            bool var_2871_transpose_y_1 = const()[name = string("op_2871_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2871_cast_fp16 = matmul(transpose_x = var_2871_transpose_x_1, transpose_y = var_2871_transpose_y_1, x = var_2869_cast_fp16, y = v_new_99_cast_fp16)[name = string("op_2871_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_101_cast_fp16 = add(x = var_2867_cast_fp16, y = var_2871_cast_fp16)[name = string("state_101_cast_fp16")];
+            tensor<int32, [4]> var_2874_begin_0 = const()[name = string("op_2874_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_2874_end_0 = const()[name = string("op_2874_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_2874_end_mask_0 = const()[name = string("op_2874_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2874_squeeze_mask_0 = const()[name = string("op_2874_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2874_cast_fp16 = slice_by_index(begin = var_2874_begin_0, end = var_2874_end_0, end_mask = var_2874_end_mask_0, squeeze_mask = var_2874_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_2874_cast_fp16")];
+            tensor<int32, [4]> var_2876_begin_0 = const()[name = string("op_2876_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_2876_end_0 = const()[name = string("op_2876_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_2876_end_mask_0 = const()[name = string("op_2876_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2876_squeeze_mask_0 = const()[name = string("op_2876_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2876_cast_fp16 = slice_by_index(begin = var_2876_begin_0, end = var_2876_end_0, end_mask = var_2876_end_mask_0, squeeze_mask = var_2876_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_2876_cast_fp16")];
+            bool var_2877_transpose_x_0 = const()[name = string("op_2877_transpose_x_0"), val = bool(false)];
+            bool var_2877_transpose_y_0 = const()[name = string("op_2877_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2877_cast_fp16 = matmul(transpose_x = var_2877_transpose_x_0, transpose_y = var_2877_transpose_y_0, x = var_2876_cast_fp16, y = state_101_cast_fp16)[name = string("op_2877_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_101_cast_fp16 = sub(x = var_2874_cast_fp16, y = var_2877_cast_fp16)[name = string("v_new_101_cast_fp16")];
+            tensor<int32, [4]> var_2880_begin_0 = const()[name = string("op_2880_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_2880_end_0 = const()[name = string("op_2880_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_2880_end_mask_0 = const()[name = string("op_2880_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2880_squeeze_mask_0 = const()[name = string("op_2880_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2880_cast_fp16 = slice_by_index(begin = var_2880_begin_0, end = var_2880_end_0, end_mask = var_2880_end_mask_0, squeeze_mask = var_2880_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_2880_cast_fp16")];
+            bool var_2881_transpose_x_0 = const()[name = string("op_2881_transpose_x_0"), val = bool(false)];
+            bool var_2881_transpose_y_0 = const()[name = string("op_2881_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2881_cast_fp16 = matmul(transpose_x = var_2881_transpose_x_0, transpose_y = var_2881_transpose_y_0, x = var_2880_cast_fp16, y = state_101_cast_fp16)[name = string("op_2881_cast_fp16")];
+            tensor<int32, [4]> var_2883_begin_0 = const()[name = string("op_2883_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_2883_end_0 = const()[name = string("op_2883_end_0"), val = tensor<int32, [4]>([32, 3, 64, 64])];
+            tensor<bool, [4]> var_2883_end_mask_0 = const()[name = string("op_2883_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2883_squeeze_mask_0 = const()[name = string("op_2883_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2883_cast_fp16 = slice_by_index(begin = var_2883_begin_0, end = var_2883_end_0, end_mask = var_2883_end_mask_0, squeeze_mask = var_2883_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_2883_cast_fp16")];
+            bool var_2884_transpose_x_0 = const()[name = string("op_2884_transpose_x_0"), val = bool(false)];
+            bool var_2884_transpose_y_0 = const()[name = string("op_2884_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2884_cast_fp16 = matmul(transpose_x = var_2884_transpose_x_0, transpose_y = var_2884_transpose_y_0, x = var_2883_cast_fp16, y = v_new_101_cast_fp16)[name = string("op_2884_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2885_cast_fp16 = add(x = var_2881_cast_fp16, y = var_2884_cast_fp16)[name = string("op_2885_cast_fp16")];
+            tensor<int32, [4]> var_2887_begin_0 = const()[name = string("op_2887_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_2887_end_0 = const()[name = string("op_2887_end_0"), val = tensor<int32, [4]>([32, 3, 1, 1])];
+            tensor<bool, [4]> var_2887_end_mask_0 = const()[name = string("op_2887_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2887_squeeze_mask_0 = const()[name = string("op_2887_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2887_cast_fp16 = slice_by_index(begin = var_2887_begin_0, end = var_2887_end_0, end_mask = var_2887_end_mask_0, squeeze_mask = var_2887_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_2887_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2888_cast_fp16 = mul(x = state_101_cast_fp16, y = var_2887_cast_fp16)[name = string("op_2888_cast_fp16")];
+            tensor<int32, [4]> var_2890_begin_0 = const()[name = string("op_2890_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_2890_end_0 = const()[name = string("op_2890_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_2890_end_mask_0 = const()[name = string("op_2890_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2890_squeeze_mask_0 = const()[name = string("op_2890_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2890_cast_fp16 = slice_by_index(begin = var_2890_begin_0, end = var_2890_end_0, end_mask = var_2890_end_mask_0, squeeze_mask = var_2890_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_2890_cast_fp16")];
+            bool var_2892_transpose_x_1 = const()[name = string("op_2892_transpose_x_1"), val = bool(true)];
+            bool var_2892_transpose_y_1 = const()[name = string("op_2892_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2892_cast_fp16 = matmul(transpose_x = var_2892_transpose_x_1, transpose_y = var_2892_transpose_y_1, x = var_2890_cast_fp16, y = v_new_101_cast_fp16)[name = string("op_2892_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_103_cast_fp16 = add(x = var_2888_cast_fp16, y = var_2892_cast_fp16)[name = string("state_103_cast_fp16")];
+            tensor<int32, [4]> var_2895_begin_0 = const()[name = string("op_2895_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_2895_end_0 = const()[name = string("op_2895_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_2895_end_mask_0 = const()[name = string("op_2895_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2895_squeeze_mask_0 = const()[name = string("op_2895_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2895_cast_fp16 = slice_by_index(begin = var_2895_begin_0, end = var_2895_end_0, end_mask = var_2895_end_mask_0, squeeze_mask = var_2895_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_2895_cast_fp16")];
+            tensor<int32, [4]> var_2897_begin_0 = const()[name = string("op_2897_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_2897_end_0 = const()[name = string("op_2897_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_2897_end_mask_0 = const()[name = string("op_2897_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2897_squeeze_mask_0 = const()[name = string("op_2897_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2897_cast_fp16 = slice_by_index(begin = var_2897_begin_0, end = var_2897_end_0, end_mask = var_2897_end_mask_0, squeeze_mask = var_2897_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_2897_cast_fp16")];
+            bool var_2898_transpose_x_0 = const()[name = string("op_2898_transpose_x_0"), val = bool(false)];
+            bool var_2898_transpose_y_0 = const()[name = string("op_2898_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2898_cast_fp16 = matmul(transpose_x = var_2898_transpose_x_0, transpose_y = var_2898_transpose_y_0, x = var_2897_cast_fp16, y = state_103_cast_fp16)[name = string("op_2898_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_103_cast_fp16 = sub(x = var_2895_cast_fp16, y = var_2898_cast_fp16)[name = string("v_new_103_cast_fp16")];
+            tensor<int32, [4]> var_2901_begin_0 = const()[name = string("op_2901_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_2901_end_0 = const()[name = string("op_2901_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_2901_end_mask_0 = const()[name = string("op_2901_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2901_squeeze_mask_0 = const()[name = string("op_2901_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2901_cast_fp16 = slice_by_index(begin = var_2901_begin_0, end = var_2901_end_0, end_mask = var_2901_end_mask_0, squeeze_mask = var_2901_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_2901_cast_fp16")];
+            bool var_2902_transpose_x_0 = const()[name = string("op_2902_transpose_x_0"), val = bool(false)];
+            bool var_2902_transpose_y_0 = const()[name = string("op_2902_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2902_cast_fp16 = matmul(transpose_x = var_2902_transpose_x_0, transpose_y = var_2902_transpose_y_0, x = var_2901_cast_fp16, y = state_103_cast_fp16)[name = string("op_2902_cast_fp16")];
+            tensor<int32, [4]> var_2904_begin_0 = const()[name = string("op_2904_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_2904_end_0 = const()[name = string("op_2904_end_0"), val = tensor<int32, [4]>([32, 4, 64, 64])];
+            tensor<bool, [4]> var_2904_end_mask_0 = const()[name = string("op_2904_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2904_squeeze_mask_0 = const()[name = string("op_2904_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2904_cast_fp16 = slice_by_index(begin = var_2904_begin_0, end = var_2904_end_0, end_mask = var_2904_end_mask_0, squeeze_mask = var_2904_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_2904_cast_fp16")];
+            bool var_2905_transpose_x_0 = const()[name = string("op_2905_transpose_x_0"), val = bool(false)];
+            bool var_2905_transpose_y_0 = const()[name = string("op_2905_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2905_cast_fp16 = matmul(transpose_x = var_2905_transpose_x_0, transpose_y = var_2905_transpose_y_0, x = var_2904_cast_fp16, y = v_new_103_cast_fp16)[name = string("op_2905_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2906_cast_fp16 = add(x = var_2902_cast_fp16, y = var_2905_cast_fp16)[name = string("op_2906_cast_fp16")];
+            tensor<int32, [4]> var_2908_begin_0 = const()[name = string("op_2908_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_2908_end_0 = const()[name = string("op_2908_end_0"), val = tensor<int32, [4]>([32, 4, 1, 1])];
+            tensor<bool, [4]> var_2908_end_mask_0 = const()[name = string("op_2908_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2908_squeeze_mask_0 = const()[name = string("op_2908_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2908_cast_fp16 = slice_by_index(begin = var_2908_begin_0, end = var_2908_end_0, end_mask = var_2908_end_mask_0, squeeze_mask = var_2908_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_2908_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2909_cast_fp16 = mul(x = state_103_cast_fp16, y = var_2908_cast_fp16)[name = string("op_2909_cast_fp16")];
+            tensor<int32, [4]> var_2911_begin_0 = const()[name = string("op_2911_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_2911_end_0 = const()[name = string("op_2911_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_2911_end_mask_0 = const()[name = string("op_2911_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2911_squeeze_mask_0 = const()[name = string("op_2911_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2911_cast_fp16 = slice_by_index(begin = var_2911_begin_0, end = var_2911_end_0, end_mask = var_2911_end_mask_0, squeeze_mask = var_2911_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_2911_cast_fp16")];
+            bool var_2913_transpose_x_1 = const()[name = string("op_2913_transpose_x_1"), val = bool(true)];
+            bool var_2913_transpose_y_1 = const()[name = string("op_2913_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2913_cast_fp16 = matmul(transpose_x = var_2913_transpose_x_1, transpose_y = var_2913_transpose_y_1, x = var_2911_cast_fp16, y = v_new_103_cast_fp16)[name = string("op_2913_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_105_cast_fp16 = add(x = var_2909_cast_fp16, y = var_2913_cast_fp16)[name = string("state_105_cast_fp16")];
+            tensor<int32, [4]> var_2916_begin_0 = const()[name = string("op_2916_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_2916_end_0 = const()[name = string("op_2916_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_2916_end_mask_0 = const()[name = string("op_2916_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2916_squeeze_mask_0 = const()[name = string("op_2916_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2916_cast_fp16 = slice_by_index(begin = var_2916_begin_0, end = var_2916_end_0, end_mask = var_2916_end_mask_0, squeeze_mask = var_2916_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_2916_cast_fp16")];
+            tensor<int32, [4]> var_2918_begin_0 = const()[name = string("op_2918_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_2918_end_0 = const()[name = string("op_2918_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_2918_end_mask_0 = const()[name = string("op_2918_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2918_squeeze_mask_0 = const()[name = string("op_2918_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2918_cast_fp16 = slice_by_index(begin = var_2918_begin_0, end = var_2918_end_0, end_mask = var_2918_end_mask_0, squeeze_mask = var_2918_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_2918_cast_fp16")];
+            bool var_2919_transpose_x_0 = const()[name = string("op_2919_transpose_x_0"), val = bool(false)];
+            bool var_2919_transpose_y_0 = const()[name = string("op_2919_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2919_cast_fp16 = matmul(transpose_x = var_2919_transpose_x_0, transpose_y = var_2919_transpose_y_0, x = var_2918_cast_fp16, y = state_105_cast_fp16)[name = string("op_2919_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_105_cast_fp16 = sub(x = var_2916_cast_fp16, y = var_2919_cast_fp16)[name = string("v_new_105_cast_fp16")];
+            tensor<int32, [4]> var_2922_begin_0 = const()[name = string("op_2922_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_2922_end_0 = const()[name = string("op_2922_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_2922_end_mask_0 = const()[name = string("op_2922_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2922_squeeze_mask_0 = const()[name = string("op_2922_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2922_cast_fp16 = slice_by_index(begin = var_2922_begin_0, end = var_2922_end_0, end_mask = var_2922_end_mask_0, squeeze_mask = var_2922_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_2922_cast_fp16")];
+            bool var_2923_transpose_x_0 = const()[name = string("op_2923_transpose_x_0"), val = bool(false)];
+            bool var_2923_transpose_y_0 = const()[name = string("op_2923_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2923_cast_fp16 = matmul(transpose_x = var_2923_transpose_x_0, transpose_y = var_2923_transpose_y_0, x = var_2922_cast_fp16, y = state_105_cast_fp16)[name = string("op_2923_cast_fp16")];
+            tensor<int32, [4]> var_2925_begin_0 = const()[name = string("op_2925_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_2925_end_0 = const()[name = string("op_2925_end_0"), val = tensor<int32, [4]>([32, 5, 64, 64])];
+            tensor<bool, [4]> var_2925_end_mask_0 = const()[name = string("op_2925_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2925_squeeze_mask_0 = const()[name = string("op_2925_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2925_cast_fp16 = slice_by_index(begin = var_2925_begin_0, end = var_2925_end_0, end_mask = var_2925_end_mask_0, squeeze_mask = var_2925_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_2925_cast_fp16")];
+            bool var_2926_transpose_x_0 = const()[name = string("op_2926_transpose_x_0"), val = bool(false)];
+            bool var_2926_transpose_y_0 = const()[name = string("op_2926_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2926_cast_fp16 = matmul(transpose_x = var_2926_transpose_x_0, transpose_y = var_2926_transpose_y_0, x = var_2925_cast_fp16, y = v_new_105_cast_fp16)[name = string("op_2926_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2927_cast_fp16 = add(x = var_2923_cast_fp16, y = var_2926_cast_fp16)[name = string("op_2927_cast_fp16")];
+            tensor<int32, [4]> var_2929_begin_0 = const()[name = string("op_2929_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_2929_end_0 = const()[name = string("op_2929_end_0"), val = tensor<int32, [4]>([32, 5, 1, 1])];
+            tensor<bool, [4]> var_2929_end_mask_0 = const()[name = string("op_2929_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2929_squeeze_mask_0 = const()[name = string("op_2929_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2929_cast_fp16 = slice_by_index(begin = var_2929_begin_0, end = var_2929_end_0, end_mask = var_2929_end_mask_0, squeeze_mask = var_2929_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_2929_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2930_cast_fp16 = mul(x = state_105_cast_fp16, y = var_2929_cast_fp16)[name = string("op_2930_cast_fp16")];
+            tensor<int32, [4]> var_2932_begin_0 = const()[name = string("op_2932_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_2932_end_0 = const()[name = string("op_2932_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_2932_end_mask_0 = const()[name = string("op_2932_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2932_squeeze_mask_0 = const()[name = string("op_2932_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2932_cast_fp16 = slice_by_index(begin = var_2932_begin_0, end = var_2932_end_0, end_mask = var_2932_end_mask_0, squeeze_mask = var_2932_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_2932_cast_fp16")];
+            bool var_2934_transpose_x_1 = const()[name = string("op_2934_transpose_x_1"), val = bool(true)];
+            bool var_2934_transpose_y_1 = const()[name = string("op_2934_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2934_cast_fp16 = matmul(transpose_x = var_2934_transpose_x_1, transpose_y = var_2934_transpose_y_1, x = var_2932_cast_fp16, y = v_new_105_cast_fp16)[name = string("op_2934_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_107_cast_fp16 = add(x = var_2930_cast_fp16, y = var_2934_cast_fp16)[name = string("state_107_cast_fp16")];
+            tensor<int32, [4]> var_2937_begin_0 = const()[name = string("op_2937_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_2937_end_0 = const()[name = string("op_2937_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_2937_end_mask_0 = const()[name = string("op_2937_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2937_squeeze_mask_0 = const()[name = string("op_2937_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2937_cast_fp16 = slice_by_index(begin = var_2937_begin_0, end = var_2937_end_0, end_mask = var_2937_end_mask_0, squeeze_mask = var_2937_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_2937_cast_fp16")];
+            tensor<int32, [4]> var_2939_begin_0 = const()[name = string("op_2939_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_2939_end_0 = const()[name = string("op_2939_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_2939_end_mask_0 = const()[name = string("op_2939_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2939_squeeze_mask_0 = const()[name = string("op_2939_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2939_cast_fp16 = slice_by_index(begin = var_2939_begin_0, end = var_2939_end_0, end_mask = var_2939_end_mask_0, squeeze_mask = var_2939_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_2939_cast_fp16")];
+            bool var_2940_transpose_x_0 = const()[name = string("op_2940_transpose_x_0"), val = bool(false)];
+            bool var_2940_transpose_y_0 = const()[name = string("op_2940_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2940_cast_fp16 = matmul(transpose_x = var_2940_transpose_x_0, transpose_y = var_2940_transpose_y_0, x = var_2939_cast_fp16, y = state_107_cast_fp16)[name = string("op_2940_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_107_cast_fp16 = sub(x = var_2937_cast_fp16, y = var_2940_cast_fp16)[name = string("v_new_107_cast_fp16")];
+            tensor<int32, [4]> var_2943_begin_0 = const()[name = string("op_2943_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_2943_end_0 = const()[name = string("op_2943_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_2943_end_mask_0 = const()[name = string("op_2943_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2943_squeeze_mask_0 = const()[name = string("op_2943_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2943_cast_fp16 = slice_by_index(begin = var_2943_begin_0, end = var_2943_end_0, end_mask = var_2943_end_mask_0, squeeze_mask = var_2943_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_2943_cast_fp16")];
+            bool var_2944_transpose_x_0 = const()[name = string("op_2944_transpose_x_0"), val = bool(false)];
+            bool var_2944_transpose_y_0 = const()[name = string("op_2944_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2944_cast_fp16 = matmul(transpose_x = var_2944_transpose_x_0, transpose_y = var_2944_transpose_y_0, x = var_2943_cast_fp16, y = state_107_cast_fp16)[name = string("op_2944_cast_fp16")];
+            tensor<int32, [4]> var_2946_begin_0 = const()[name = string("op_2946_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_2946_end_0 = const()[name = string("op_2946_end_0"), val = tensor<int32, [4]>([32, 6, 64, 64])];
+            tensor<bool, [4]> var_2946_end_mask_0 = const()[name = string("op_2946_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2946_squeeze_mask_0 = const()[name = string("op_2946_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2946_cast_fp16 = slice_by_index(begin = var_2946_begin_0, end = var_2946_end_0, end_mask = var_2946_end_mask_0, squeeze_mask = var_2946_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_2946_cast_fp16")];
+            bool var_2947_transpose_x_0 = const()[name = string("op_2947_transpose_x_0"), val = bool(false)];
+            bool var_2947_transpose_y_0 = const()[name = string("op_2947_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2947_cast_fp16 = matmul(transpose_x = var_2947_transpose_x_0, transpose_y = var_2947_transpose_y_0, x = var_2946_cast_fp16, y = v_new_107_cast_fp16)[name = string("op_2947_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2948_cast_fp16 = add(x = var_2944_cast_fp16, y = var_2947_cast_fp16)[name = string("op_2948_cast_fp16")];
+            tensor<int32, [4]> var_2950_begin_0 = const()[name = string("op_2950_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_2950_end_0 = const()[name = string("op_2950_end_0"), val = tensor<int32, [4]>([32, 6, 1, 1])];
+            tensor<bool, [4]> var_2950_end_mask_0 = const()[name = string("op_2950_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2950_squeeze_mask_0 = const()[name = string("op_2950_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2950_cast_fp16 = slice_by_index(begin = var_2950_begin_0, end = var_2950_end_0, end_mask = var_2950_end_mask_0, squeeze_mask = var_2950_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_2950_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2951_cast_fp16 = mul(x = state_107_cast_fp16, y = var_2950_cast_fp16)[name = string("op_2951_cast_fp16")];
+            tensor<int32, [4]> var_2953_begin_0 = const()[name = string("op_2953_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_2953_end_0 = const()[name = string("op_2953_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_2953_end_mask_0 = const()[name = string("op_2953_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2953_squeeze_mask_0 = const()[name = string("op_2953_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2953_cast_fp16 = slice_by_index(begin = var_2953_begin_0, end = var_2953_end_0, end_mask = var_2953_end_mask_0, squeeze_mask = var_2953_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_2953_cast_fp16")];
+            bool var_2955_transpose_x_1 = const()[name = string("op_2955_transpose_x_1"), val = bool(true)];
+            bool var_2955_transpose_y_1 = const()[name = string("op_2955_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2955_cast_fp16 = matmul(transpose_x = var_2955_transpose_x_1, transpose_y = var_2955_transpose_y_1, x = var_2953_cast_fp16, y = v_new_107_cast_fp16)[name = string("op_2955_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_109_cast_fp16 = add(x = var_2951_cast_fp16, y = var_2955_cast_fp16)[name = string("state_109_cast_fp16")];
+            tensor<int32, [4]> var_2958_begin_0 = const()[name = string("op_2958_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_2958_end_0 = const()[name = string("op_2958_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_2958_end_mask_0 = const()[name = string("op_2958_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2958_squeeze_mask_0 = const()[name = string("op_2958_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2958_cast_fp16 = slice_by_index(begin = var_2958_begin_0, end = var_2958_end_0, end_mask = var_2958_end_mask_0, squeeze_mask = var_2958_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_2958_cast_fp16")];
+            tensor<int32, [4]> var_2960_begin_0 = const()[name = string("op_2960_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_2960_end_0 = const()[name = string("op_2960_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_2960_end_mask_0 = const()[name = string("op_2960_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2960_squeeze_mask_0 = const()[name = string("op_2960_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2960_cast_fp16 = slice_by_index(begin = var_2960_begin_0, end = var_2960_end_0, end_mask = var_2960_end_mask_0, squeeze_mask = var_2960_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_2960_cast_fp16")];
+            bool var_2961_transpose_x_0 = const()[name = string("op_2961_transpose_x_0"), val = bool(false)];
+            bool var_2961_transpose_y_0 = const()[name = string("op_2961_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2961_cast_fp16 = matmul(transpose_x = var_2961_transpose_x_0, transpose_y = var_2961_transpose_y_0, x = var_2960_cast_fp16, y = state_109_cast_fp16)[name = string("op_2961_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_109_cast_fp16 = sub(x = var_2958_cast_fp16, y = var_2961_cast_fp16)[name = string("v_new_109_cast_fp16")];
+            tensor<int32, [4]> var_2964_begin_0 = const()[name = string("op_2964_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_2964_end_0 = const()[name = string("op_2964_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_2964_end_mask_0 = const()[name = string("op_2964_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2964_squeeze_mask_0 = const()[name = string("op_2964_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2964_cast_fp16 = slice_by_index(begin = var_2964_begin_0, end = var_2964_end_0, end_mask = var_2964_end_mask_0, squeeze_mask = var_2964_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_2964_cast_fp16")];
+            bool var_2965_transpose_x_0 = const()[name = string("op_2965_transpose_x_0"), val = bool(false)];
+            bool var_2965_transpose_y_0 = const()[name = string("op_2965_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2965_cast_fp16 = matmul(transpose_x = var_2965_transpose_x_0, transpose_y = var_2965_transpose_y_0, x = var_2964_cast_fp16, y = state_109_cast_fp16)[name = string("op_2965_cast_fp16")];
+            tensor<int32, [4]> var_2967_begin_0 = const()[name = string("op_2967_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_2967_end_0 = const()[name = string("op_2967_end_0"), val = tensor<int32, [4]>([32, 7, 64, 64])];
+            tensor<bool, [4]> var_2967_end_mask_0 = const()[name = string("op_2967_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2967_squeeze_mask_0 = const()[name = string("op_2967_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2967_cast_fp16 = slice_by_index(begin = var_2967_begin_0, end = var_2967_end_0, end_mask = var_2967_end_mask_0, squeeze_mask = var_2967_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_2967_cast_fp16")];
+            bool var_2968_transpose_x_0 = const()[name = string("op_2968_transpose_x_0"), val = bool(false)];
+            bool var_2968_transpose_y_0 = const()[name = string("op_2968_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2968_cast_fp16 = matmul(transpose_x = var_2968_transpose_x_0, transpose_y = var_2968_transpose_y_0, x = var_2967_cast_fp16, y = v_new_109_cast_fp16)[name = string("op_2968_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2969_cast_fp16 = add(x = var_2965_cast_fp16, y = var_2968_cast_fp16)[name = string("op_2969_cast_fp16")];
+            tensor<int32, [4]> var_2971_begin_0 = const()[name = string("op_2971_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_2971_end_0 = const()[name = string("op_2971_end_0"), val = tensor<int32, [4]>([32, 7, 1, 1])];
+            tensor<bool, [4]> var_2971_end_mask_0 = const()[name = string("op_2971_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2971_squeeze_mask_0 = const()[name = string("op_2971_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2971_cast_fp16 = slice_by_index(begin = var_2971_begin_0, end = var_2971_end_0, end_mask = var_2971_end_mask_0, squeeze_mask = var_2971_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_2971_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2972_cast_fp16 = mul(x = state_109_cast_fp16, y = var_2971_cast_fp16)[name = string("op_2972_cast_fp16")];
+            tensor<int32, [4]> var_2974_begin_0 = const()[name = string("op_2974_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_2974_end_0 = const()[name = string("op_2974_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_2974_end_mask_0 = const()[name = string("op_2974_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2974_squeeze_mask_0 = const()[name = string("op_2974_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2974_cast_fp16 = slice_by_index(begin = var_2974_begin_0, end = var_2974_end_0, end_mask = var_2974_end_mask_0, squeeze_mask = var_2974_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_2974_cast_fp16")];
+            bool var_2976_transpose_x_1 = const()[name = string("op_2976_transpose_x_1"), val = bool(true)];
+            bool var_2976_transpose_y_1 = const()[name = string("op_2976_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2976_cast_fp16 = matmul(transpose_x = var_2976_transpose_x_1, transpose_y = var_2976_transpose_y_1, x = var_2974_cast_fp16, y = v_new_109_cast_fp16)[name = string("op_2976_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_111_cast_fp16 = add(x = var_2972_cast_fp16, y = var_2976_cast_fp16)[name = string("state_111_cast_fp16")];
+            tensor<int32, [4]> var_2979_begin_0 = const()[name = string("op_2979_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_2979_end_0 = const()[name = string("op_2979_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_2979_end_mask_0 = const()[name = string("op_2979_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2979_squeeze_mask_0 = const()[name = string("op_2979_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2979_cast_fp16 = slice_by_index(begin = var_2979_begin_0, end = var_2979_end_0, end_mask = var_2979_end_mask_0, squeeze_mask = var_2979_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_2979_cast_fp16")];
+            tensor<int32, [4]> var_2981_begin_0 = const()[name = string("op_2981_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_2981_end_0 = const()[name = string("op_2981_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_2981_end_mask_0 = const()[name = string("op_2981_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2981_squeeze_mask_0 = const()[name = string("op_2981_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2981_cast_fp16 = slice_by_index(begin = var_2981_begin_0, end = var_2981_end_0, end_mask = var_2981_end_mask_0, squeeze_mask = var_2981_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_2981_cast_fp16")];
+            bool var_2982_transpose_x_0 = const()[name = string("op_2982_transpose_x_0"), val = bool(false)];
+            bool var_2982_transpose_y_0 = const()[name = string("op_2982_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2982_cast_fp16 = matmul(transpose_x = var_2982_transpose_x_0, transpose_y = var_2982_transpose_y_0, x = var_2981_cast_fp16, y = state_111_cast_fp16)[name = string("op_2982_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_111_cast_fp16 = sub(x = var_2979_cast_fp16, y = var_2982_cast_fp16)[name = string("v_new_111_cast_fp16")];
+            tensor<int32, [4]> var_2985_begin_0 = const()[name = string("op_2985_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_2985_end_0 = const()[name = string("op_2985_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_2985_end_mask_0 = const()[name = string("op_2985_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2985_squeeze_mask_0 = const()[name = string("op_2985_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2985_cast_fp16 = slice_by_index(begin = var_2985_begin_0, end = var_2985_end_0, end_mask = var_2985_end_mask_0, squeeze_mask = var_2985_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_2985_cast_fp16")];
+            bool var_2986_transpose_x_0 = const()[name = string("op_2986_transpose_x_0"), val = bool(false)];
+            bool var_2986_transpose_y_0 = const()[name = string("op_2986_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2986_cast_fp16 = matmul(transpose_x = var_2986_transpose_x_0, transpose_y = var_2986_transpose_y_0, x = var_2985_cast_fp16, y = state_111_cast_fp16)[name = string("op_2986_cast_fp16")];
+            tensor<int32, [4]> var_2988_begin_0 = const()[name = string("op_2988_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_2988_end_0 = const()[name = string("op_2988_end_0"), val = tensor<int32, [4]>([32, 8, 64, 64])];
+            tensor<bool, [4]> var_2988_end_mask_0 = const()[name = string("op_2988_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2988_squeeze_mask_0 = const()[name = string("op_2988_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2988_cast_fp16 = slice_by_index(begin = var_2988_begin_0, end = var_2988_end_0, end_mask = var_2988_end_mask_0, squeeze_mask = var_2988_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_2988_cast_fp16")];
+            bool var_2989_transpose_x_0 = const()[name = string("op_2989_transpose_x_0"), val = bool(false)];
+            bool var_2989_transpose_y_0 = const()[name = string("op_2989_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2989_cast_fp16 = matmul(transpose_x = var_2989_transpose_x_0, transpose_y = var_2989_transpose_y_0, x = var_2988_cast_fp16, y = v_new_111_cast_fp16)[name = string("op_2989_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2990_cast_fp16 = add(x = var_2986_cast_fp16, y = var_2989_cast_fp16)[name = string("op_2990_cast_fp16")];
+            tensor<int32, [4]> var_2992_begin_0 = const()[name = string("op_2992_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_2992_end_0 = const()[name = string("op_2992_end_0"), val = tensor<int32, [4]>([32, 8, 1, 1])];
+            tensor<bool, [4]> var_2992_end_mask_0 = const()[name = string("op_2992_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2992_squeeze_mask_0 = const()[name = string("op_2992_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2992_cast_fp16 = slice_by_index(begin = var_2992_begin_0, end = var_2992_end_0, end_mask = var_2992_end_mask_0, squeeze_mask = var_2992_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_2992_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2993_cast_fp16 = mul(x = state_111_cast_fp16, y = var_2992_cast_fp16)[name = string("op_2993_cast_fp16")];
+            tensor<int32, [4]> var_2995_begin_0 = const()[name = string("op_2995_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_2995_end_0 = const()[name = string("op_2995_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_2995_end_mask_0 = const()[name = string("op_2995_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2995_squeeze_mask_0 = const()[name = string("op_2995_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2995_cast_fp16 = slice_by_index(begin = var_2995_begin_0, end = var_2995_end_0, end_mask = var_2995_end_mask_0, squeeze_mask = var_2995_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_2995_cast_fp16")];
+            bool var_2997_transpose_x_1 = const()[name = string("op_2997_transpose_x_1"), val = bool(true)];
+            bool var_2997_transpose_y_1 = const()[name = string("op_2997_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2997_cast_fp16 = matmul(transpose_x = var_2997_transpose_x_1, transpose_y = var_2997_transpose_y_1, x = var_2995_cast_fp16, y = v_new_111_cast_fp16)[name = string("op_2997_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_113_cast_fp16 = add(x = var_2993_cast_fp16, y = var_2997_cast_fp16)[name = string("state_113_cast_fp16")];
+            tensor<int32, [4]> var_3000_begin_0 = const()[name = string("op_3000_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_3000_end_0 = const()[name = string("op_3000_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_3000_end_mask_0 = const()[name = string("op_3000_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3000_squeeze_mask_0 = const()[name = string("op_3000_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3000_cast_fp16 = slice_by_index(begin = var_3000_begin_0, end = var_3000_end_0, end_mask = var_3000_end_mask_0, squeeze_mask = var_3000_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_3000_cast_fp16")];
+            tensor<int32, [4]> var_3002_begin_0 = const()[name = string("op_3002_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_3002_end_0 = const()[name = string("op_3002_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_3002_end_mask_0 = const()[name = string("op_3002_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3002_squeeze_mask_0 = const()[name = string("op_3002_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3002_cast_fp16 = slice_by_index(begin = var_3002_begin_0, end = var_3002_end_0, end_mask = var_3002_end_mask_0, squeeze_mask = var_3002_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_3002_cast_fp16")];
+            bool var_3003_transpose_x_0 = const()[name = string("op_3003_transpose_x_0"), val = bool(false)];
+            bool var_3003_transpose_y_0 = const()[name = string("op_3003_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3003_cast_fp16 = matmul(transpose_x = var_3003_transpose_x_0, transpose_y = var_3003_transpose_y_0, x = var_3002_cast_fp16, y = state_113_cast_fp16)[name = string("op_3003_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_113_cast_fp16 = sub(x = var_3000_cast_fp16, y = var_3003_cast_fp16)[name = string("v_new_113_cast_fp16")];
+            tensor<int32, [4]> var_3006_begin_0 = const()[name = string("op_3006_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_3006_end_0 = const()[name = string("op_3006_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_3006_end_mask_0 = const()[name = string("op_3006_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3006_squeeze_mask_0 = const()[name = string("op_3006_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3006_cast_fp16 = slice_by_index(begin = var_3006_begin_0, end = var_3006_end_0, end_mask = var_3006_end_mask_0, squeeze_mask = var_3006_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_3006_cast_fp16")];
+            bool var_3007_transpose_x_0 = const()[name = string("op_3007_transpose_x_0"), val = bool(false)];
+            bool var_3007_transpose_y_0 = const()[name = string("op_3007_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3007_cast_fp16 = matmul(transpose_x = var_3007_transpose_x_0, transpose_y = var_3007_transpose_y_0, x = var_3006_cast_fp16, y = state_113_cast_fp16)[name = string("op_3007_cast_fp16")];
+            tensor<int32, [4]> var_3009_begin_0 = const()[name = string("op_3009_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_3009_end_0 = const()[name = string("op_3009_end_0"), val = tensor<int32, [4]>([32, 9, 64, 64])];
+            tensor<bool, [4]> var_3009_end_mask_0 = const()[name = string("op_3009_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3009_squeeze_mask_0 = const()[name = string("op_3009_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3009_cast_fp16 = slice_by_index(begin = var_3009_begin_0, end = var_3009_end_0, end_mask = var_3009_end_mask_0, squeeze_mask = var_3009_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_3009_cast_fp16")];
+            bool var_3010_transpose_x_0 = const()[name = string("op_3010_transpose_x_0"), val = bool(false)];
+            bool var_3010_transpose_y_0 = const()[name = string("op_3010_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3010_cast_fp16 = matmul(transpose_x = var_3010_transpose_x_0, transpose_y = var_3010_transpose_y_0, x = var_3009_cast_fp16, y = v_new_113_cast_fp16)[name = string("op_3010_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3011_cast_fp16 = add(x = var_3007_cast_fp16, y = var_3010_cast_fp16)[name = string("op_3011_cast_fp16")];
+            tensor<int32, [4]> var_3013_begin_0 = const()[name = string("op_3013_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_3013_end_0 = const()[name = string("op_3013_end_0"), val = tensor<int32, [4]>([32, 9, 1, 1])];
+            tensor<bool, [4]> var_3013_end_mask_0 = const()[name = string("op_3013_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3013_squeeze_mask_0 = const()[name = string("op_3013_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3013_cast_fp16 = slice_by_index(begin = var_3013_begin_0, end = var_3013_end_0, end_mask = var_3013_end_mask_0, squeeze_mask = var_3013_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_3013_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3014_cast_fp16 = mul(x = state_113_cast_fp16, y = var_3013_cast_fp16)[name = string("op_3014_cast_fp16")];
+            tensor<int32, [4]> var_3016_begin_0 = const()[name = string("op_3016_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_3016_end_0 = const()[name = string("op_3016_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_3016_end_mask_0 = const()[name = string("op_3016_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3016_squeeze_mask_0 = const()[name = string("op_3016_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3016_cast_fp16 = slice_by_index(begin = var_3016_begin_0, end = var_3016_end_0, end_mask = var_3016_end_mask_0, squeeze_mask = var_3016_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_3016_cast_fp16")];
+            bool var_3018_transpose_x_1 = const()[name = string("op_3018_transpose_x_1"), val = bool(true)];
+            bool var_3018_transpose_y_1 = const()[name = string("op_3018_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3018_cast_fp16 = matmul(transpose_x = var_3018_transpose_x_1, transpose_y = var_3018_transpose_y_1, x = var_3016_cast_fp16, y = v_new_113_cast_fp16)[name = string("op_3018_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_115_cast_fp16 = add(x = var_3014_cast_fp16, y = var_3018_cast_fp16)[name = string("state_115_cast_fp16")];
+            tensor<int32, [4]> var_3021_begin_0 = const()[name = string("op_3021_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_3021_end_0 = const()[name = string("op_3021_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_3021_end_mask_0 = const()[name = string("op_3021_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3021_squeeze_mask_0 = const()[name = string("op_3021_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3021_cast_fp16 = slice_by_index(begin = var_3021_begin_0, end = var_3021_end_0, end_mask = var_3021_end_mask_0, squeeze_mask = var_3021_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_3021_cast_fp16")];
+            tensor<int32, [4]> var_3023_begin_0 = const()[name = string("op_3023_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_3023_end_0 = const()[name = string("op_3023_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_3023_end_mask_0 = const()[name = string("op_3023_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3023_squeeze_mask_0 = const()[name = string("op_3023_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3023_cast_fp16 = slice_by_index(begin = var_3023_begin_0, end = var_3023_end_0, end_mask = var_3023_end_mask_0, squeeze_mask = var_3023_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_3023_cast_fp16")];
+            bool var_3024_transpose_x_0 = const()[name = string("op_3024_transpose_x_0"), val = bool(false)];
+            bool var_3024_transpose_y_0 = const()[name = string("op_3024_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3024_cast_fp16 = matmul(transpose_x = var_3024_transpose_x_0, transpose_y = var_3024_transpose_y_0, x = var_3023_cast_fp16, y = state_115_cast_fp16)[name = string("op_3024_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_115_cast_fp16 = sub(x = var_3021_cast_fp16, y = var_3024_cast_fp16)[name = string("v_new_115_cast_fp16")];
+            tensor<int32, [4]> var_3027_begin_0 = const()[name = string("op_3027_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_3027_end_0 = const()[name = string("op_3027_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_3027_end_mask_0 = const()[name = string("op_3027_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3027_squeeze_mask_0 = const()[name = string("op_3027_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3027_cast_fp16 = slice_by_index(begin = var_3027_begin_0, end = var_3027_end_0, end_mask = var_3027_end_mask_0, squeeze_mask = var_3027_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_3027_cast_fp16")];
+            bool var_3028_transpose_x_0 = const()[name = string("op_3028_transpose_x_0"), val = bool(false)];
+            bool var_3028_transpose_y_0 = const()[name = string("op_3028_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3028_cast_fp16 = matmul(transpose_x = var_3028_transpose_x_0, transpose_y = var_3028_transpose_y_0, x = var_3027_cast_fp16, y = state_115_cast_fp16)[name = string("op_3028_cast_fp16")];
+            tensor<int32, [4]> var_3030_begin_0 = const()[name = string("op_3030_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_3030_end_0 = const()[name = string("op_3030_end_0"), val = tensor<int32, [4]>([32, 10, 64, 64])];
+            tensor<bool, [4]> var_3030_end_mask_0 = const()[name = string("op_3030_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3030_squeeze_mask_0 = const()[name = string("op_3030_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3030_cast_fp16 = slice_by_index(begin = var_3030_begin_0, end = var_3030_end_0, end_mask = var_3030_end_mask_0, squeeze_mask = var_3030_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_3030_cast_fp16")];
+            bool var_3031_transpose_x_0 = const()[name = string("op_3031_transpose_x_0"), val = bool(false)];
+            bool var_3031_transpose_y_0 = const()[name = string("op_3031_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3031_cast_fp16 = matmul(transpose_x = var_3031_transpose_x_0, transpose_y = var_3031_transpose_y_0, x = var_3030_cast_fp16, y = v_new_115_cast_fp16)[name = string("op_3031_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3032_cast_fp16 = add(x = var_3028_cast_fp16, y = var_3031_cast_fp16)[name = string("op_3032_cast_fp16")];
+            tensor<int32, [4]> var_3034_begin_0 = const()[name = string("op_3034_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_3034_end_0 = const()[name = string("op_3034_end_0"), val = tensor<int32, [4]>([32, 10, 1, 1])];
+            tensor<bool, [4]> var_3034_end_mask_0 = const()[name = string("op_3034_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3034_squeeze_mask_0 = const()[name = string("op_3034_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3034_cast_fp16 = slice_by_index(begin = var_3034_begin_0, end = var_3034_end_0, end_mask = var_3034_end_mask_0, squeeze_mask = var_3034_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_3034_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3035_cast_fp16 = mul(x = state_115_cast_fp16, y = var_3034_cast_fp16)[name = string("op_3035_cast_fp16")];
+            tensor<int32, [4]> var_3037_begin_0 = const()[name = string("op_3037_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_3037_end_0 = const()[name = string("op_3037_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_3037_end_mask_0 = const()[name = string("op_3037_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3037_squeeze_mask_0 = const()[name = string("op_3037_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3037_cast_fp16 = slice_by_index(begin = var_3037_begin_0, end = var_3037_end_0, end_mask = var_3037_end_mask_0, squeeze_mask = var_3037_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_3037_cast_fp16")];
+            bool var_3039_transpose_x_1 = const()[name = string("op_3039_transpose_x_1"), val = bool(true)];
+            bool var_3039_transpose_y_1 = const()[name = string("op_3039_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3039_cast_fp16 = matmul(transpose_x = var_3039_transpose_x_1, transpose_y = var_3039_transpose_y_1, x = var_3037_cast_fp16, y = v_new_115_cast_fp16)[name = string("op_3039_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_117_cast_fp16 = add(x = var_3035_cast_fp16, y = var_3039_cast_fp16)[name = string("state_117_cast_fp16")];
+            tensor<int32, [4]> var_3042_begin_0 = const()[name = string("op_3042_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_3042_end_0 = const()[name = string("op_3042_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_3042_end_mask_0 = const()[name = string("op_3042_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3042_squeeze_mask_0 = const()[name = string("op_3042_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3042_cast_fp16 = slice_by_index(begin = var_3042_begin_0, end = var_3042_end_0, end_mask = var_3042_end_mask_0, squeeze_mask = var_3042_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_3042_cast_fp16")];
+            tensor<int32, [4]> var_3044_begin_0 = const()[name = string("op_3044_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_3044_end_0 = const()[name = string("op_3044_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_3044_end_mask_0 = const()[name = string("op_3044_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3044_squeeze_mask_0 = const()[name = string("op_3044_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3044_cast_fp16 = slice_by_index(begin = var_3044_begin_0, end = var_3044_end_0, end_mask = var_3044_end_mask_0, squeeze_mask = var_3044_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_3044_cast_fp16")];
+            bool var_3045_transpose_x_0 = const()[name = string("op_3045_transpose_x_0"), val = bool(false)];
+            bool var_3045_transpose_y_0 = const()[name = string("op_3045_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3045_cast_fp16 = matmul(transpose_x = var_3045_transpose_x_0, transpose_y = var_3045_transpose_y_0, x = var_3044_cast_fp16, y = state_117_cast_fp16)[name = string("op_3045_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_117_cast_fp16 = sub(x = var_3042_cast_fp16, y = var_3045_cast_fp16)[name = string("v_new_117_cast_fp16")];
+            tensor<int32, [4]> var_3048_begin_0 = const()[name = string("op_3048_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_3048_end_0 = const()[name = string("op_3048_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_3048_end_mask_0 = const()[name = string("op_3048_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3048_squeeze_mask_0 = const()[name = string("op_3048_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3048_cast_fp16 = slice_by_index(begin = var_3048_begin_0, end = var_3048_end_0, end_mask = var_3048_end_mask_0, squeeze_mask = var_3048_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_3048_cast_fp16")];
+            bool var_3049_transpose_x_0 = const()[name = string("op_3049_transpose_x_0"), val = bool(false)];
+            bool var_3049_transpose_y_0 = const()[name = string("op_3049_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3049_cast_fp16 = matmul(transpose_x = var_3049_transpose_x_0, transpose_y = var_3049_transpose_y_0, x = var_3048_cast_fp16, y = state_117_cast_fp16)[name = string("op_3049_cast_fp16")];
+            tensor<int32, [4]> var_3051_begin_0 = const()[name = string("op_3051_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_3051_end_0 = const()[name = string("op_3051_end_0"), val = tensor<int32, [4]>([32, 11, 64, 64])];
+            tensor<bool, [4]> var_3051_end_mask_0 = const()[name = string("op_3051_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3051_squeeze_mask_0 = const()[name = string("op_3051_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3051_cast_fp16 = slice_by_index(begin = var_3051_begin_0, end = var_3051_end_0, end_mask = var_3051_end_mask_0, squeeze_mask = var_3051_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_3051_cast_fp16")];
+            bool var_3052_transpose_x_0 = const()[name = string("op_3052_transpose_x_0"), val = bool(false)];
+            bool var_3052_transpose_y_0 = const()[name = string("op_3052_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3052_cast_fp16 = matmul(transpose_x = var_3052_transpose_x_0, transpose_y = var_3052_transpose_y_0, x = var_3051_cast_fp16, y = v_new_117_cast_fp16)[name = string("op_3052_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3053_cast_fp16 = add(x = var_3049_cast_fp16, y = var_3052_cast_fp16)[name = string("op_3053_cast_fp16")];
+            tensor<int32, [4]> var_3055_begin_0 = const()[name = string("op_3055_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_3055_end_0 = const()[name = string("op_3055_end_0"), val = tensor<int32, [4]>([32, 11, 1, 1])];
+            tensor<bool, [4]> var_3055_end_mask_0 = const()[name = string("op_3055_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3055_squeeze_mask_0 = const()[name = string("op_3055_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3055_cast_fp16 = slice_by_index(begin = var_3055_begin_0, end = var_3055_end_0, end_mask = var_3055_end_mask_0, squeeze_mask = var_3055_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_3055_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3056_cast_fp16 = mul(x = state_117_cast_fp16, y = var_3055_cast_fp16)[name = string("op_3056_cast_fp16")];
+            tensor<int32, [4]> var_3058_begin_0 = const()[name = string("op_3058_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_3058_end_0 = const()[name = string("op_3058_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_3058_end_mask_0 = const()[name = string("op_3058_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3058_squeeze_mask_0 = const()[name = string("op_3058_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3058_cast_fp16 = slice_by_index(begin = var_3058_begin_0, end = var_3058_end_0, end_mask = var_3058_end_mask_0, squeeze_mask = var_3058_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_3058_cast_fp16")];
+            bool var_3060_transpose_x_1 = const()[name = string("op_3060_transpose_x_1"), val = bool(true)];
+            bool var_3060_transpose_y_1 = const()[name = string("op_3060_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3060_cast_fp16 = matmul(transpose_x = var_3060_transpose_x_1, transpose_y = var_3060_transpose_y_1, x = var_3058_cast_fp16, y = v_new_117_cast_fp16)[name = string("op_3060_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_119_cast_fp16 = add(x = var_3056_cast_fp16, y = var_3060_cast_fp16)[name = string("state_119_cast_fp16")];
+            tensor<int32, [4]> var_3063_begin_0 = const()[name = string("op_3063_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_3063_end_0 = const()[name = string("op_3063_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_3063_end_mask_0 = const()[name = string("op_3063_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3063_squeeze_mask_0 = const()[name = string("op_3063_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3063_cast_fp16 = slice_by_index(begin = var_3063_begin_0, end = var_3063_end_0, end_mask = var_3063_end_mask_0, squeeze_mask = var_3063_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_3063_cast_fp16")];
+            tensor<int32, [4]> var_3065_begin_0 = const()[name = string("op_3065_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_3065_end_0 = const()[name = string("op_3065_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_3065_end_mask_0 = const()[name = string("op_3065_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3065_squeeze_mask_0 = const()[name = string("op_3065_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3065_cast_fp16 = slice_by_index(begin = var_3065_begin_0, end = var_3065_end_0, end_mask = var_3065_end_mask_0, squeeze_mask = var_3065_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_3065_cast_fp16")];
+            bool var_3066_transpose_x_0 = const()[name = string("op_3066_transpose_x_0"), val = bool(false)];
+            bool var_3066_transpose_y_0 = const()[name = string("op_3066_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3066_cast_fp16 = matmul(transpose_x = var_3066_transpose_x_0, transpose_y = var_3066_transpose_y_0, x = var_3065_cast_fp16, y = state_119_cast_fp16)[name = string("op_3066_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_119_cast_fp16 = sub(x = var_3063_cast_fp16, y = var_3066_cast_fp16)[name = string("v_new_119_cast_fp16")];
+            tensor<int32, [4]> var_3069_begin_0 = const()[name = string("op_3069_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_3069_end_0 = const()[name = string("op_3069_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_3069_end_mask_0 = const()[name = string("op_3069_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3069_squeeze_mask_0 = const()[name = string("op_3069_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3069_cast_fp16 = slice_by_index(begin = var_3069_begin_0, end = var_3069_end_0, end_mask = var_3069_end_mask_0, squeeze_mask = var_3069_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_3069_cast_fp16")];
+            bool var_3070_transpose_x_0 = const()[name = string("op_3070_transpose_x_0"), val = bool(false)];
+            bool var_3070_transpose_y_0 = const()[name = string("op_3070_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3070_cast_fp16 = matmul(transpose_x = var_3070_transpose_x_0, transpose_y = var_3070_transpose_y_0, x = var_3069_cast_fp16, y = state_119_cast_fp16)[name = string("op_3070_cast_fp16")];
+            tensor<int32, [4]> var_3072_begin_0 = const()[name = string("op_3072_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_3072_end_0 = const()[name = string("op_3072_end_0"), val = tensor<int32, [4]>([32, 12, 64, 64])];
+            tensor<bool, [4]> var_3072_end_mask_0 = const()[name = string("op_3072_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3072_squeeze_mask_0 = const()[name = string("op_3072_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3072_cast_fp16 = slice_by_index(begin = var_3072_begin_0, end = var_3072_end_0, end_mask = var_3072_end_mask_0, squeeze_mask = var_3072_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_3072_cast_fp16")];
+            bool var_3073_transpose_x_0 = const()[name = string("op_3073_transpose_x_0"), val = bool(false)];
+            bool var_3073_transpose_y_0 = const()[name = string("op_3073_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3073_cast_fp16 = matmul(transpose_x = var_3073_transpose_x_0, transpose_y = var_3073_transpose_y_0, x = var_3072_cast_fp16, y = v_new_119_cast_fp16)[name = string("op_3073_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3074_cast_fp16 = add(x = var_3070_cast_fp16, y = var_3073_cast_fp16)[name = string("op_3074_cast_fp16")];
+            tensor<int32, [4]> var_3076_begin_0 = const()[name = string("op_3076_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_3076_end_0 = const()[name = string("op_3076_end_0"), val = tensor<int32, [4]>([32, 12, 1, 1])];
+            tensor<bool, [4]> var_3076_end_mask_0 = const()[name = string("op_3076_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3076_squeeze_mask_0 = const()[name = string("op_3076_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3076_cast_fp16 = slice_by_index(begin = var_3076_begin_0, end = var_3076_end_0, end_mask = var_3076_end_mask_0, squeeze_mask = var_3076_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_3076_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3077_cast_fp16 = mul(x = state_119_cast_fp16, y = var_3076_cast_fp16)[name = string("op_3077_cast_fp16")];
+            tensor<int32, [4]> var_3079_begin_0 = const()[name = string("op_3079_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_3079_end_0 = const()[name = string("op_3079_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_3079_end_mask_0 = const()[name = string("op_3079_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3079_squeeze_mask_0 = const()[name = string("op_3079_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3079_cast_fp16 = slice_by_index(begin = var_3079_begin_0, end = var_3079_end_0, end_mask = var_3079_end_mask_0, squeeze_mask = var_3079_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_3079_cast_fp16")];
+            bool var_3081_transpose_x_1 = const()[name = string("op_3081_transpose_x_1"), val = bool(true)];
+            bool var_3081_transpose_y_1 = const()[name = string("op_3081_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3081_cast_fp16 = matmul(transpose_x = var_3081_transpose_x_1, transpose_y = var_3081_transpose_y_1, x = var_3079_cast_fp16, y = v_new_119_cast_fp16)[name = string("op_3081_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_121_cast_fp16 = add(x = var_3077_cast_fp16, y = var_3081_cast_fp16)[name = string("state_121_cast_fp16")];
+            tensor<int32, [4]> var_3084_begin_0 = const()[name = string("op_3084_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_3084_end_0 = const()[name = string("op_3084_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_3084_end_mask_0 = const()[name = string("op_3084_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3084_squeeze_mask_0 = const()[name = string("op_3084_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3084_cast_fp16 = slice_by_index(begin = var_3084_begin_0, end = var_3084_end_0, end_mask = var_3084_end_mask_0, squeeze_mask = var_3084_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_3084_cast_fp16")];
+            tensor<int32, [4]> var_3086_begin_0 = const()[name = string("op_3086_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_3086_end_0 = const()[name = string("op_3086_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_3086_end_mask_0 = const()[name = string("op_3086_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3086_squeeze_mask_0 = const()[name = string("op_3086_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3086_cast_fp16 = slice_by_index(begin = var_3086_begin_0, end = var_3086_end_0, end_mask = var_3086_end_mask_0, squeeze_mask = var_3086_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_3086_cast_fp16")];
+            bool var_3087_transpose_x_0 = const()[name = string("op_3087_transpose_x_0"), val = bool(false)];
+            bool var_3087_transpose_y_0 = const()[name = string("op_3087_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3087_cast_fp16 = matmul(transpose_x = var_3087_transpose_x_0, transpose_y = var_3087_transpose_y_0, x = var_3086_cast_fp16, y = state_121_cast_fp16)[name = string("op_3087_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_121_cast_fp16 = sub(x = var_3084_cast_fp16, y = var_3087_cast_fp16)[name = string("v_new_121_cast_fp16")];
+            tensor<int32, [4]> var_3090_begin_0 = const()[name = string("op_3090_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_3090_end_0 = const()[name = string("op_3090_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_3090_end_mask_0 = const()[name = string("op_3090_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3090_squeeze_mask_0 = const()[name = string("op_3090_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3090_cast_fp16 = slice_by_index(begin = var_3090_begin_0, end = var_3090_end_0, end_mask = var_3090_end_mask_0, squeeze_mask = var_3090_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_3090_cast_fp16")];
+            bool var_3091_transpose_x_0 = const()[name = string("op_3091_transpose_x_0"), val = bool(false)];
+            bool var_3091_transpose_y_0 = const()[name = string("op_3091_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3091_cast_fp16 = matmul(transpose_x = var_3091_transpose_x_0, transpose_y = var_3091_transpose_y_0, x = var_3090_cast_fp16, y = state_121_cast_fp16)[name = string("op_3091_cast_fp16")];
+            tensor<int32, [4]> var_3093_begin_0 = const()[name = string("op_3093_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_3093_end_0 = const()[name = string("op_3093_end_0"), val = tensor<int32, [4]>([32, 13, 64, 64])];
+            tensor<bool, [4]> var_3093_end_mask_0 = const()[name = string("op_3093_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3093_squeeze_mask_0 = const()[name = string("op_3093_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3093_cast_fp16 = slice_by_index(begin = var_3093_begin_0, end = var_3093_end_0, end_mask = var_3093_end_mask_0, squeeze_mask = var_3093_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_3093_cast_fp16")];
+            bool var_3094_transpose_x_0 = const()[name = string("op_3094_transpose_x_0"), val = bool(false)];
+            bool var_3094_transpose_y_0 = const()[name = string("op_3094_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3094_cast_fp16 = matmul(transpose_x = var_3094_transpose_x_0, transpose_y = var_3094_transpose_y_0, x = var_3093_cast_fp16, y = v_new_121_cast_fp16)[name = string("op_3094_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3095_cast_fp16 = add(x = var_3091_cast_fp16, y = var_3094_cast_fp16)[name = string("op_3095_cast_fp16")];
+            tensor<int32, [4]> var_3097_begin_0 = const()[name = string("op_3097_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_3097_end_0 = const()[name = string("op_3097_end_0"), val = tensor<int32, [4]>([32, 13, 1, 1])];
+            tensor<bool, [4]> var_3097_end_mask_0 = const()[name = string("op_3097_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3097_squeeze_mask_0 = const()[name = string("op_3097_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3097_cast_fp16 = slice_by_index(begin = var_3097_begin_0, end = var_3097_end_0, end_mask = var_3097_end_mask_0, squeeze_mask = var_3097_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_3097_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3098_cast_fp16 = mul(x = state_121_cast_fp16, y = var_3097_cast_fp16)[name = string("op_3098_cast_fp16")];
+            tensor<int32, [4]> var_3100_begin_0 = const()[name = string("op_3100_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_3100_end_0 = const()[name = string("op_3100_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_3100_end_mask_0 = const()[name = string("op_3100_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3100_squeeze_mask_0 = const()[name = string("op_3100_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3100_cast_fp16 = slice_by_index(begin = var_3100_begin_0, end = var_3100_end_0, end_mask = var_3100_end_mask_0, squeeze_mask = var_3100_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_3100_cast_fp16")];
+            bool var_3102_transpose_x_1 = const()[name = string("op_3102_transpose_x_1"), val = bool(true)];
+            bool var_3102_transpose_y_1 = const()[name = string("op_3102_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3102_cast_fp16 = matmul(transpose_x = var_3102_transpose_x_1, transpose_y = var_3102_transpose_y_1, x = var_3100_cast_fp16, y = v_new_121_cast_fp16)[name = string("op_3102_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_123_cast_fp16 = add(x = var_3098_cast_fp16, y = var_3102_cast_fp16)[name = string("state_123_cast_fp16")];
+            tensor<int32, [4]> var_3105_begin_0 = const()[name = string("op_3105_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_3105_end_0 = const()[name = string("op_3105_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_3105_end_mask_0 = const()[name = string("op_3105_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3105_squeeze_mask_0 = const()[name = string("op_3105_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3105_cast_fp16 = slice_by_index(begin = var_3105_begin_0, end = var_3105_end_0, end_mask = var_3105_end_mask_0, squeeze_mask = var_3105_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_3105_cast_fp16")];
+            tensor<int32, [4]> var_3107_begin_0 = const()[name = string("op_3107_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_3107_end_0 = const()[name = string("op_3107_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_3107_end_mask_0 = const()[name = string("op_3107_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3107_squeeze_mask_0 = const()[name = string("op_3107_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3107_cast_fp16 = slice_by_index(begin = var_3107_begin_0, end = var_3107_end_0, end_mask = var_3107_end_mask_0, squeeze_mask = var_3107_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_3107_cast_fp16")];
+            bool var_3108_transpose_x_0 = const()[name = string("op_3108_transpose_x_0"), val = bool(false)];
+            bool var_3108_transpose_y_0 = const()[name = string("op_3108_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3108_cast_fp16 = matmul(transpose_x = var_3108_transpose_x_0, transpose_y = var_3108_transpose_y_0, x = var_3107_cast_fp16, y = state_123_cast_fp16)[name = string("op_3108_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_123_cast_fp16 = sub(x = var_3105_cast_fp16, y = var_3108_cast_fp16)[name = string("v_new_123_cast_fp16")];
+            tensor<int32, [4]> var_3111_begin_0 = const()[name = string("op_3111_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_3111_end_0 = const()[name = string("op_3111_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_3111_end_mask_0 = const()[name = string("op_3111_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3111_squeeze_mask_0 = const()[name = string("op_3111_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3111_cast_fp16 = slice_by_index(begin = var_3111_begin_0, end = var_3111_end_0, end_mask = var_3111_end_mask_0, squeeze_mask = var_3111_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_3111_cast_fp16")];
+            bool var_3112_transpose_x_0 = const()[name = string("op_3112_transpose_x_0"), val = bool(false)];
+            bool var_3112_transpose_y_0 = const()[name = string("op_3112_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3112_cast_fp16 = matmul(transpose_x = var_3112_transpose_x_0, transpose_y = var_3112_transpose_y_0, x = var_3111_cast_fp16, y = state_123_cast_fp16)[name = string("op_3112_cast_fp16")];
+            tensor<int32, [4]> var_3114_begin_0 = const()[name = string("op_3114_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_3114_end_0 = const()[name = string("op_3114_end_0"), val = tensor<int32, [4]>([32, 14, 64, 64])];
+            tensor<bool, [4]> var_3114_end_mask_0 = const()[name = string("op_3114_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3114_squeeze_mask_0 = const()[name = string("op_3114_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3114_cast_fp16 = slice_by_index(begin = var_3114_begin_0, end = var_3114_end_0, end_mask = var_3114_end_mask_0, squeeze_mask = var_3114_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_3114_cast_fp16")];
+            bool var_3115_transpose_x_0 = const()[name = string("op_3115_transpose_x_0"), val = bool(false)];
+            bool var_3115_transpose_y_0 = const()[name = string("op_3115_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3115_cast_fp16 = matmul(transpose_x = var_3115_transpose_x_0, transpose_y = var_3115_transpose_y_0, x = var_3114_cast_fp16, y = v_new_123_cast_fp16)[name = string("op_3115_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3116_cast_fp16 = add(x = var_3112_cast_fp16, y = var_3115_cast_fp16)[name = string("op_3116_cast_fp16")];
+            tensor<int32, [4]> var_3118_begin_0 = const()[name = string("op_3118_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_3118_end_0 = const()[name = string("op_3118_end_0"), val = tensor<int32, [4]>([32, 14, 1, 1])];
+            tensor<bool, [4]> var_3118_end_mask_0 = const()[name = string("op_3118_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3118_squeeze_mask_0 = const()[name = string("op_3118_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3118_cast_fp16 = slice_by_index(begin = var_3118_begin_0, end = var_3118_end_0, end_mask = var_3118_end_mask_0, squeeze_mask = var_3118_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_3118_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3119_cast_fp16 = mul(x = state_123_cast_fp16, y = var_3118_cast_fp16)[name = string("op_3119_cast_fp16")];
+            tensor<int32, [4]> var_3121_begin_0 = const()[name = string("op_3121_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_3121_end_0 = const()[name = string("op_3121_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_3121_end_mask_0 = const()[name = string("op_3121_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3121_squeeze_mask_0 = const()[name = string("op_3121_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3121_cast_fp16 = slice_by_index(begin = var_3121_begin_0, end = var_3121_end_0, end_mask = var_3121_end_mask_0, squeeze_mask = var_3121_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_3121_cast_fp16")];
+            bool var_3123_transpose_x_1 = const()[name = string("op_3123_transpose_x_1"), val = bool(true)];
+            bool var_3123_transpose_y_1 = const()[name = string("op_3123_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3123_cast_fp16 = matmul(transpose_x = var_3123_transpose_x_1, transpose_y = var_3123_transpose_y_1, x = var_3121_cast_fp16, y = v_new_123_cast_fp16)[name = string("op_3123_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_125_cast_fp16 = add(x = var_3119_cast_fp16, y = var_3123_cast_fp16)[name = string("state_125_cast_fp16")];
+            tensor<int32, [4]> var_3126_begin_0 = const()[name = string("op_3126_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_3126_end_0 = const()[name = string("op_3126_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_3126_end_mask_0 = const()[name = string("op_3126_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3126_squeeze_mask_0 = const()[name = string("op_3126_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3126_cast_fp16 = slice_by_index(begin = var_3126_begin_0, end = var_3126_end_0, end_mask = var_3126_end_mask_0, squeeze_mask = var_3126_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_3126_cast_fp16")];
+            tensor<int32, [4]> var_3128_begin_0 = const()[name = string("op_3128_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_3128_end_0 = const()[name = string("op_3128_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_3128_end_mask_0 = const()[name = string("op_3128_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3128_squeeze_mask_0 = const()[name = string("op_3128_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3128_cast_fp16 = slice_by_index(begin = var_3128_begin_0, end = var_3128_end_0, end_mask = var_3128_end_mask_0, squeeze_mask = var_3128_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_3128_cast_fp16")];
+            bool var_3129_transpose_x_0 = const()[name = string("op_3129_transpose_x_0"), val = bool(false)];
+            bool var_3129_transpose_y_0 = const()[name = string("op_3129_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3129_cast_fp16 = matmul(transpose_x = var_3129_transpose_x_0, transpose_y = var_3129_transpose_y_0, x = var_3128_cast_fp16, y = state_125_cast_fp16)[name = string("op_3129_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_125_cast_fp16 = sub(x = var_3126_cast_fp16, y = var_3129_cast_fp16)[name = string("v_new_125_cast_fp16")];
+            tensor<int32, [4]> var_3132_begin_0 = const()[name = string("op_3132_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_3132_end_0 = const()[name = string("op_3132_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_3132_end_mask_0 = const()[name = string("op_3132_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3132_squeeze_mask_0 = const()[name = string("op_3132_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3132_cast_fp16 = slice_by_index(begin = var_3132_begin_0, end = var_3132_end_0, end_mask = var_3132_end_mask_0, squeeze_mask = var_3132_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_3132_cast_fp16")];
+            bool var_3133_transpose_x_0 = const()[name = string("op_3133_transpose_x_0"), val = bool(false)];
+            bool var_3133_transpose_y_0 = const()[name = string("op_3133_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3133_cast_fp16 = matmul(transpose_x = var_3133_transpose_x_0, transpose_y = var_3133_transpose_y_0, x = var_3132_cast_fp16, y = state_125_cast_fp16)[name = string("op_3133_cast_fp16")];
+            tensor<int32, [4]> var_3135_begin_0 = const()[name = string("op_3135_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_3135_end_0 = const()[name = string("op_3135_end_0"), val = tensor<int32, [4]>([32, 15, 64, 64])];
+            tensor<bool, [4]> var_3135_end_mask_0 = const()[name = string("op_3135_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3135_squeeze_mask_0 = const()[name = string("op_3135_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3135_cast_fp16 = slice_by_index(begin = var_3135_begin_0, end = var_3135_end_0, end_mask = var_3135_end_mask_0, squeeze_mask = var_3135_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_3135_cast_fp16")];
+            bool var_3136_transpose_x_0 = const()[name = string("op_3136_transpose_x_0"), val = bool(false)];
+            bool var_3136_transpose_y_0 = const()[name = string("op_3136_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3136_cast_fp16 = matmul(transpose_x = var_3136_transpose_x_0, transpose_y = var_3136_transpose_y_0, x = var_3135_cast_fp16, y = v_new_125_cast_fp16)[name = string("op_3136_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3137_cast_fp16 = add(x = var_3133_cast_fp16, y = var_3136_cast_fp16)[name = string("op_3137_cast_fp16")];
+            tensor<int32, [4]> var_3139_begin_0 = const()[name = string("op_3139_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_3139_end_0 = const()[name = string("op_3139_end_0"), val = tensor<int32, [4]>([32, 15, 1, 1])];
+            tensor<bool, [4]> var_3139_end_mask_0 = const()[name = string("op_3139_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3139_squeeze_mask_0 = const()[name = string("op_3139_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3139_cast_fp16 = slice_by_index(begin = var_3139_begin_0, end = var_3139_end_0, end_mask = var_3139_end_mask_0, squeeze_mask = var_3139_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_3139_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3140_cast_fp16 = mul(x = state_125_cast_fp16, y = var_3139_cast_fp16)[name = string("op_3140_cast_fp16")];
+            tensor<int32, [4]> var_3142_begin_0 = const()[name = string("op_3142_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_3142_end_0 = const()[name = string("op_3142_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_3142_end_mask_0 = const()[name = string("op_3142_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3142_squeeze_mask_0 = const()[name = string("op_3142_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3142_cast_fp16 = slice_by_index(begin = var_3142_begin_0, end = var_3142_end_0, end_mask = var_3142_end_mask_0, squeeze_mask = var_3142_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_3142_cast_fp16")];
+            bool var_3144_transpose_x_1 = const()[name = string("op_3144_transpose_x_1"), val = bool(true)];
+            bool var_3144_transpose_y_1 = const()[name = string("op_3144_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3144_cast_fp16 = matmul(transpose_x = var_3144_transpose_x_1, transpose_y = var_3144_transpose_y_1, x = var_3142_cast_fp16, y = v_new_125_cast_fp16)[name = string("op_3144_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_127_cast_fp16 = add(x = var_3140_cast_fp16, y = var_3144_cast_fp16)[name = string("state_127_cast_fp16")];
+            tensor<int32, [4]> var_3147_begin_0 = const()[name = string("op_3147_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_3147_end_0 = const()[name = string("op_3147_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_3147_end_mask_0 = const()[name = string("op_3147_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3147_squeeze_mask_0 = const()[name = string("op_3147_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3147_cast_fp16 = slice_by_index(begin = var_3147_begin_0, end = var_3147_end_0, end_mask = var_3147_end_mask_0, squeeze_mask = var_3147_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_3147_cast_fp16")];
+            tensor<int32, [4]> var_3149_begin_0 = const()[name = string("op_3149_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_3149_end_0 = const()[name = string("op_3149_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_3149_end_mask_0 = const()[name = string("op_3149_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3149_squeeze_mask_0 = const()[name = string("op_3149_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3149_cast_fp16 = slice_by_index(begin = var_3149_begin_0, end = var_3149_end_0, end_mask = var_3149_end_mask_0, squeeze_mask = var_3149_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_3149_cast_fp16")];
+            bool var_3150_transpose_x_0 = const()[name = string("op_3150_transpose_x_0"), val = bool(false)];
+            bool var_3150_transpose_y_0 = const()[name = string("op_3150_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3150_cast_fp16 = matmul(transpose_x = var_3150_transpose_x_0, transpose_y = var_3150_transpose_y_0, x = var_3149_cast_fp16, y = state_127_cast_fp16)[name = string("op_3150_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_127_cast_fp16 = sub(x = var_3147_cast_fp16, y = var_3150_cast_fp16)[name = string("v_new_127_cast_fp16")];
+            tensor<int32, [4]> var_3153_begin_0 = const()[name = string("op_3153_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_3153_end_0 = const()[name = string("op_3153_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_3153_end_mask_0 = const()[name = string("op_3153_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3153_squeeze_mask_0 = const()[name = string("op_3153_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3153_cast_fp16 = slice_by_index(begin = var_3153_begin_0, end = var_3153_end_0, end_mask = var_3153_end_mask_0, squeeze_mask = var_3153_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_3153_cast_fp16")];
+            bool var_3154_transpose_x_0 = const()[name = string("op_3154_transpose_x_0"), val = bool(false)];
+            bool var_3154_transpose_y_0 = const()[name = string("op_3154_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3154_cast_fp16 = matmul(transpose_x = var_3154_transpose_x_0, transpose_y = var_3154_transpose_y_0, x = var_3153_cast_fp16, y = state_127_cast_fp16)[name = string("op_3154_cast_fp16")];
+            tensor<int32, [4]> var_3156_begin_0 = const()[name = string("op_3156_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_3156_end_0 = const()[name = string("op_3156_end_0"), val = tensor<int32, [4]>([32, 16, 64, 64])];
+            tensor<bool, [4]> var_3156_end_mask_0 = const()[name = string("op_3156_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3156_squeeze_mask_0 = const()[name = string("op_3156_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3156_cast_fp16 = slice_by_index(begin = var_3156_begin_0, end = var_3156_end_0, end_mask = var_3156_end_mask_0, squeeze_mask = var_3156_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_3156_cast_fp16")];
+            bool var_3157_transpose_x_0 = const()[name = string("op_3157_transpose_x_0"), val = bool(false)];
+            bool var_3157_transpose_y_0 = const()[name = string("op_3157_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3157_cast_fp16 = matmul(transpose_x = var_3157_transpose_x_0, transpose_y = var_3157_transpose_y_0, x = var_3156_cast_fp16, y = v_new_127_cast_fp16)[name = string("op_3157_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3158_cast_fp16 = add(x = var_3154_cast_fp16, y = var_3157_cast_fp16)[name = string("op_3158_cast_fp16")];
+            int32 var_3160_axis_0 = const()[name = string("op_3160_axis_0"), val = int32(1)];
+            tensor<fp16, [32, 16, 64, 128]> var_3160_cast_fp16 = stack(axis = var_3160_axis_0, values = (var_2843_cast_fp16, var_2864_cast_fp16, var_2885_cast_fp16, var_2906_cast_fp16, var_2927_cast_fp16, var_2948_cast_fp16, var_2969_cast_fp16, var_2990_cast_fp16, var_3011_cast_fp16, var_3032_cast_fp16, var_3053_cast_fp16, var_3074_cast_fp16, var_3095_cast_fp16, var_3116_cast_fp16, var_3137_cast_fp16, var_3158_cast_fp16))[name = string("op_3160_cast_fp16")];
+            tensor<int32, [3]> var_3161 = const()[name = string("op_3161"), val = tensor<int32, [3]>([32, 1024, 128])];
+            tensor<fp16, [32, 1024, 128]> var_3162_cast_fp16 = reshape(shape = var_3161, x = var_3160_cast_fp16)[name = string("op_3162_cast_fp16")];
+            tensor<int32, [3]> var_3163 = const()[name = string("op_3163"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> input_63_begin_0 = const()[name = string("input_63_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> input_63_end_0 = const()[name = string("input_63_end_0"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<bool, [4]> input_63_end_mask_0 = const()[name = string("input_63_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> input_63_squeeze_mask_0 = const()[name = string("input_63_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [1024, 32, 128]> input_63_cast_fp16 = slice_by_index(begin = input_63_begin_0, end = input_63_end_0, end_mask = input_63_end_mask_0, squeeze_mask = input_63_squeeze_mask_0, x = z_7_cast_fp16)[name = string("input_63_cast_fp16")];
+            fp16 var_2479_promoted_1_to_fp16 = const()[name = string("op_2479_promoted_1_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1024, 32, 128]> x_117_cast_fp16 = transpose(perm = var_3163, x = var_3162_cast_fp16)[name = string("transpose_20")];
+            tensor<fp16, [1024, 32, 128]> var_3167_cast_fp16 = pow(x = x_117_cast_fp16, y = var_2479_promoted_1_to_fp16)[name = string("op_3167_cast_fp16")];
+            tensor<int32, [1]> var_3169_axes_0 = const()[name = string("op_3169_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_3169_keep_dims_0 = const()[name = string("op_3169_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1024, 32, 1]> var_3169_cast_fp16 = reduce_mean(axes = var_3169_axes_0, keep_dims = var_3169_keep_dims_0, x = var_3167_cast_fp16)[name = string("op_3169_cast_fp16")];
+            fp16 var_3170_to_fp16 = const()[name = string("op_3170_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1024, 32, 1]> var_3171_cast_fp16 = add(x = var_3169_cast_fp16, y = var_3170_to_fp16)[name = string("op_3171_cast_fp16")];
+            fp32 var_3172_epsilon_0 = const()[name = string("op_3172_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1024, 32, 1]> var_3172_cast_fp16 = rsqrt(epsilon = var_3172_epsilon_0, x = var_3171_cast_fp16)[name = string("op_3172_cast_fp16")];
+            tensor<fp16, [1024, 32, 128]> x_119_cast_fp16 = mul(x = x_117_cast_fp16, y = var_3172_cast_fp16)[name = string("x_119_cast_fp16")];
+            tensor<fp16, [128]> layers_4_linear_attn_norm_weight_to_fp16 = const()[name = string("layers_4_linear_attn_norm_weight_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(687867776)))];
+            tensor<fp16, [1024, 32, 128]> var_3174_cast_fp16 = mul(x = layers_4_linear_attn_norm_weight_to_fp16, y = x_119_cast_fp16)[name = string("op_3174_cast_fp16")];
+            tensor<fp16, [1024, 32, 128]> var_3175_cast_fp16 = silu(x = input_63_cast_fp16)[name = string("op_3175_cast_fp16")];
+            tensor<fp16, [1024, 32, 128]> core_7_cast_fp16 = mul(x = var_3174_cast_fp16, y = var_3175_cast_fp16)[name = string("core_7_cast_fp16")];
+            tensor<int32, [3]> var_3177 = const()[name = string("op_3177"), val = tensor<int32, [3]>([1, 1024, 4096])];
+            tensor<fp16, [1, 1024, 4096]> input_65_cast_fp16 = reshape(shape = var_3177, x = core_7_cast_fp16)[name = string("input_65_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> linear_35_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers_4_linear_attn_out_proj_weight_cast_fp16, x = input_65_cast_fp16)[name = string("linear_35_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> x_121_cast_fp16 = add(x = x_99_cast_fp16, y = linear_35_cast_fp16)[name = string("x_121_cast_fp16")];
+            fp16 var_2479_promoted_2_to_fp16 = const()[name = string("op_2479_promoted_2_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 1024, 2560]> var_3183_cast_fp16 = pow(x = x_121_cast_fp16, y = var_2479_promoted_2_to_fp16)[name = string("op_3183_cast_fp16")];
+            tensor<int32, [1]> var_3185_axes_0 = const()[name = string("op_3185_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_3185_keep_dims_0 = const()[name = string("op_3185_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 1024, 1]> var_3185_cast_fp16 = reduce_mean(axes = var_3185_axes_0, keep_dims = var_3185_keep_dims_0, x = var_3183_cast_fp16)[name = string("op_3185_cast_fp16")];
+            fp16 var_3186_to_fp16 = const()[name = string("op_3186_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 1024, 1]> var_3187_cast_fp16 = add(x = var_3185_cast_fp16, y = var_3186_to_fp16)[name = string("op_3187_cast_fp16")];
+            fp32 var_3188_epsilon_0 = const()[name = string("op_3188_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 1024, 1]> var_3188_cast_fp16 = rsqrt(epsilon = var_3188_epsilon_0, x = var_3187_cast_fp16)[name = string("op_3188_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> x_123_cast_fp16 = mul(x = x_121_cast_fp16, y = var_3188_cast_fp16)[name = string("x_123_cast_fp16")];
+            tensor<fp16, [2560]> var_3191_to_fp16 = const()[name = string("op_3191_to_fp16"), val = tensor<fp16, [2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(687868096)))];
+            tensor<fp16, [1, 1024, 2560]> input_67_cast_fp16 = mul(x = x_123_cast_fp16, y = var_3191_to_fp16)[name = string("input_67_cast_fp16")];
+            tensor<fp16, [1, 1024, 9216]> linear_36_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = layers_4_mlp_gate_proj_weight_cast_fp16, x = input_67_cast_fp16)[name = string("linear_36_cast_fp16")];
+            tensor<fp16, [1, 1024, 9216]> var_3198_cast_fp16 = silu(x = linear_36_cast_fp16)[name = string("op_3198_cast_fp16")];
+            tensor<fp16, [1, 1024, 9216]> linear_37_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = layers_4_mlp_up_proj_weight_cast_fp16, x = input_67_cast_fp16)[name = string("linear_37_cast_fp16")];
+            tensor<fp16, [1, 1024, 9216]> input_71_cast_fp16 = mul(x = var_3198_cast_fp16, y = linear_37_cast_fp16)[name = string("input_71_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> linear_38_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers_4_mlp_down_proj_weight_cast_fp16, x = input_71_cast_fp16)[name = string("linear_38_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> x_125_cast_fp16 = add(x = x_121_cast_fp16, y = linear_38_cast_fp16)[name = string("x_125_cast_fp16")];
+            int32 var_3218 = const()[name = string("op_3218"), val = int32(-2)];
+            int32 var_3233 = const()[name = string("op_3233"), val = int32(-1)];
+            fp16 var_3232_promoted_to_fp16 = const()[name = string("op_3232_promoted_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 1024, 2560]> var_3242_cast_fp16 = pow(x = x_125_cast_fp16, y = var_3232_promoted_to_fp16)[name = string("op_3242_cast_fp16")];
+            tensor<int32, [1]> var_3244_axes_0 = const()[name = string("op_3244_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_3244_keep_dims_0 = const()[name = string("op_3244_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 1024, 1]> var_3244_cast_fp16 = reduce_mean(axes = var_3244_axes_0, keep_dims = var_3244_keep_dims_0, x = var_3242_cast_fp16)[name = string("op_3244_cast_fp16")];
+            fp16 var_3245_to_fp16 = const()[name = string("op_3245_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 1024, 1]> var_3246_cast_fp16 = add(x = var_3244_cast_fp16, y = var_3245_to_fp16)[name = string("op_3246_cast_fp16")];
+            fp32 var_3247_epsilon_0 = const()[name = string("op_3247_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 1024, 1]> var_3247_cast_fp16 = rsqrt(epsilon = var_3247_epsilon_0, x = var_3246_cast_fp16)[name = string("op_3247_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> x_127_cast_fp16 = mul(x = x_125_cast_fp16, y = var_3247_cast_fp16)[name = string("x_127_cast_fp16")];
+            tensor<fp16, [2560]> var_3250_to_fp16 = const()[name = string("op_3250_to_fp16"), val = tensor<fp16, [2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(687873280)))];
+            tensor<fp16, [1, 1024, 2560]> x_129_cast_fp16 = mul(x = x_127_cast_fp16, y = var_3250_to_fp16)[name = string("x_129_cast_fp16")];
+            tensor<fp16, [1, 1024, 8192]> linear_39_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = layers_5_linear_attn_in_proj_qkv_weight_cast_fp16, x = x_129_cast_fp16)[name = string("linear_39_cast_fp16")];
+            tensor<int32, [3]> input_73_perm_0 = const()[name = string("input_73_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
+            tensor<int32, [6]> input_75_pad_0 = const()[name = string("input_75_pad_0"), val = tensor<int32, [6]>([0, 0, 0, 0, 3, 0])];
+            string input_75_mode_0 = const()[name = string("input_75_mode_0"), val = string("constant")];
+            fp16 const_54_to_fp16 = const()[name = string("const_54_to_fp16"), val = fp16(0x0p+0)];
+            tensor<fp16, [1, 8192, 1024]> input_73_cast_fp16 = transpose(perm = input_73_perm_0, x = linear_39_cast_fp16)[name = string("transpose_19")];
+            tensor<fp16, [1, 8192, 1027]> input_75_cast_fp16 = pad(constant_val = const_54_to_fp16, mode = input_75_mode_0, pad = input_75_pad_0, x = input_73_cast_fp16)[name = string("input_75_cast_fp16")];
+            string input_77_pad_type_0 = const()[name = string("input_77_pad_type_0"), val = string("valid")];
+            int32 input_77_groups_0 = const()[name = string("input_77_groups_0"), val = int32(8192)];
+            tensor<int32, [1]> input_77_strides_0 = const()[name = string("input_77_strides_0"), val = tensor<int32, [1]>([1])];
+            tensor<int32, [2]> input_77_pad_0 = const()[name = string("input_77_pad_0"), val = tensor<int32, [2]>([0, 0])];
+            tensor<int32, [1]> input_77_dilations_0 = const()[name = string("input_77_dilations_0"), val = tensor<int32, [1]>([1])];
+            tensor<fp16, [8192, 1, 4]> layers_5_linear_attn_conv1d_weight_to_fp16 = const()[name = string("layers_5_linear_attn_conv1d_weight_to_fp16"), val = tensor<fp16, [8192, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(687878464)))];
+            tensor<fp16, [1, 8192, 1024]> input_77_cast_fp16 = conv(dilations = input_77_dilations_0, groups = input_77_groups_0, pad = input_77_pad_0, pad_type = input_77_pad_type_0, strides = input_77_strides_0, weight = layers_5_linear_attn_conv1d_weight_to_fp16, x = input_75_cast_fp16)[name = string("input_77_cast_fp16")];
+            tensor<fp16, [1, 8192, 1024]> var_3285_cast_fp16 = silu(x = input_77_cast_fp16)[name = string("op_3285_cast_fp16")];
+            tensor<int32, [3]> qkv_9_perm_0 = const()[name = string("qkv_9_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
+            tensor<int32, [3]> var_3287_begin_0 = const()[name = string("op_3287_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_3287_end_0 = const()[name = string("op_3287_end_0"), val = tensor<int32, [3]>([1, 1024, 2048])];
+            tensor<bool, [3]> var_3287_end_mask_0 = const()[name = string("op_3287_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
+            tensor<fp16, [1, 1024, 8192]> qkv_9_cast_fp16 = transpose(perm = qkv_9_perm_0, x = var_3285_cast_fp16)[name = string("transpose_18")];
+            tensor<fp16, [1, 1024, 2048]> var_3287_cast_fp16 = slice_by_index(begin = var_3287_begin_0, end = var_3287_end_0, end_mask = var_3287_end_mask_0, x = qkv_9_cast_fp16)[name = string("op_3287_cast_fp16")];
+            tensor<int32, [4]> var_3288 = const()[name = string("op_3288"), val = tensor<int32, [4]>([1, 1024, 16, 128])];
+            tensor<fp16, [1, 1024, 16, 128]> q_45_cast_fp16 = reshape(shape = var_3288, x = var_3287_cast_fp16)[name = string("q_45_cast_fp16")];
+            tensor<int32, [3]> var_3290_begin_0 = const()[name = string("op_3290_begin_0"), val = tensor<int32, [3]>([0, 0, 2048])];
+            tensor<int32, [3]> var_3290_end_0 = const()[name = string("op_3290_end_0"), val = tensor<int32, [3]>([1, 1024, 4096])];
+            tensor<bool, [3]> var_3290_end_mask_0 = const()[name = string("op_3290_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
+            tensor<fp16, [1, 1024, 2048]> var_3290_cast_fp16 = slice_by_index(begin = var_3290_begin_0, end = var_3290_end_0, end_mask = var_3290_end_mask_0, x = qkv_9_cast_fp16)[name = string("op_3290_cast_fp16")];
+            tensor<int32, [4]> var_3291 = const()[name = string("op_3291"), val = tensor<int32, [4]>([1, 1024, 16, 128])];
+            tensor<fp16, [1, 1024, 16, 128]> k_39_cast_fp16 = reshape(shape = var_3291, x = var_3290_cast_fp16)[name = string("k_39_cast_fp16")];
+            tensor<int32, [3]> var_3293_begin_0 = const()[name = string("op_3293_begin_0"), val = tensor<int32, [3]>([0, 0, 4096])];
+            tensor<int32, [3]> var_3293_end_0 = const()[name = string("op_3293_end_0"), val = tensor<int32, [3]>([1, 1024, 8192])];
+            tensor<bool, [3]> var_3293_end_mask_0 = const()[name = string("op_3293_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
+            tensor<fp16, [1, 1024, 4096]> var_3293_cast_fp16 = slice_by_index(begin = var_3293_begin_0, end = var_3293_end_0, end_mask = var_3293_end_mask_0, x = qkv_9_cast_fp16)[name = string("op_3293_cast_fp16")];
+            tensor<int32, [4]> var_3294 = const()[name = string("op_3294"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<fp16, [1, 1024, 32, 128]> v_21_cast_fp16 = reshape(shape = var_3294, x = var_3293_cast_fp16)[name = string("v_21_cast_fp16")];
+            tensor<fp16, [1, 1024, 4096]> linear_40_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = layers_5_linear_attn_in_proj_z_weight_cast_fp16, x = x_129_cast_fp16)[name = string("linear_40_cast_fp16")];
+            tensor<int32, [4]> var_3298 = const()[name = string("op_3298"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<fp16, [1, 1024, 32, 128]> z_9_cast_fp16 = reshape(shape = var_3298, x = linear_40_cast_fp16)[name = string("z_9_cast_fp16")];
+            tensor<fp16, [32]> linear_41_bias_0_to_fp16 = const()[name = string("linear_41_bias_0_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(687944064)))];
+            tensor<fp16, [1, 1024, 32]> linear_41_cast_fp16 = linear(bias = linear_41_bias_0_to_fp16, weight = layers_5_linear_attn_in_proj_b_weight_cast_fp16, x = x_129_cast_fp16)[name = string("linear_41_cast_fp16")];
+            tensor<fp16, [1, 1024, 32]> beta_17_cast_fp16 = sigmoid(x = linear_41_cast_fp16)[name = string("beta_17_cast_fp16")];
+            tensor<fp16, [32]> linear_42_bias_0_to_fp16 = const()[name = string("linear_42_bias_0_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(687944192)))];
+            tensor<fp16, [1, 1024, 32]> linear_42_cast_fp16 = linear(bias = linear_42_bias_0_to_fp16, weight = layers_5_linear_attn_in_proj_a_weight_cast_fp16, x = x_129_cast_fp16)[name = string("linear_42_cast_fp16")];
+            tensor<fp16, [32]> layers_5_linear_attn_dt_bias_to_fp16 = const()[name = string("layers_5_linear_attn_dt_bias_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(687944320)))];
+            tensor<fp16, [1, 1024, 32]> var_3307_cast_fp16 = add(x = linear_42_cast_fp16, y = layers_5_linear_attn_dt_bias_to_fp16)[name = string("op_3307_cast_fp16")];
+            tensor<fp16, [1, 1024, 32]> var_3308_cast_fp16 = softplus(x = var_3307_cast_fp16)[name = string("op_3308_cast_fp16")];
+            tensor<fp16, [32]> var_3304_to_fp16 = const()[name = string("op_3304_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(687944448)))];
+            tensor<fp16, [1, 1024, 32]> g_17_cast_fp16 = mul(x = var_3304_to_fp16, y = var_3308_cast_fp16)[name = string("g_17_cast_fp16")];
+            tensor<fp16, [1, 1024, 16, 128]> var_3310_cast_fp16 = mul(x = q_45_cast_fp16, y = q_45_cast_fp16)[name = string("op_3310_cast_fp16")];
+            tensor<int32, [1]> var_3312_axes_0 = const()[name = string("op_3312_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_3312_keep_dims_0 = const()[name = string("op_3312_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 1024, 16, 1]> var_3312_cast_fp16 = reduce_sum(axes = var_3312_axes_0, keep_dims = var_3312_keep_dims_0, x = var_3310_cast_fp16)[name = string("op_3312_cast_fp16")];
+            fp16 var_3313_to_fp16 = const()[name = string("op_3313_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 1024, 16, 1]> var_3314_cast_fp16 = add(x = var_3312_cast_fp16, y = var_3313_to_fp16)[name = string("op_3314_cast_fp16")];
+            fp32 var_3315_epsilon_0 = const()[name = string("op_3315_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 1024, 16, 1]> var_3315_cast_fp16 = rsqrt(epsilon = var_3315_epsilon_0, x = var_3314_cast_fp16)[name = string("op_3315_cast_fp16")];
+            tensor<fp16, [1, 1024, 16, 128]> q_47_cast_fp16 = mul(x = q_45_cast_fp16, y = var_3315_cast_fp16)[name = string("q_47_cast_fp16")];
+            tensor<fp16, [1, 1024, 16, 128]> var_3317_cast_fp16 = mul(x = k_39_cast_fp16, y = k_39_cast_fp16)[name = string("op_3317_cast_fp16")];
+            tensor<int32, [1]> var_3319_axes_0 = const()[name = string("op_3319_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_3319_keep_dims_0 = const()[name = string("op_3319_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 1024, 16, 1]> var_3319_cast_fp16 = reduce_sum(axes = var_3319_axes_0, keep_dims = var_3319_keep_dims_0, x = var_3317_cast_fp16)[name = string("op_3319_cast_fp16")];
+            fp16 var_3320_to_fp16 = const()[name = string("op_3320_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 1024, 16, 1]> var_3321_cast_fp16 = add(x = var_3319_cast_fp16, y = var_3320_to_fp16)[name = string("op_3321_cast_fp16")];
+            fp32 var_3322_epsilon_0 = const()[name = string("op_3322_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 1024, 16, 1]> var_3322_cast_fp16 = rsqrt(epsilon = var_3322_epsilon_0, x = var_3321_cast_fp16)[name = string("op_3322_cast_fp16")];
+            tensor<fp16, [1, 1024, 16, 128]> k_41_cast_fp16 = mul(x = k_39_cast_fp16, y = var_3322_cast_fp16)[name = string("k_41_cast_fp16")];
+            tensor<int32, [1]> var_3327_axes_0 = const()[name = string("op_3327_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [1, 1024, 16, 1, 128]> var_3327_cast_fp16 = expand_dims(axes = var_3327_axes_0, x = q_47_cast_fp16)[name = string("op_3327_cast_fp16")];
+            tensor<int32, [5]> var_3329_reps_0 = const()[name = string("op_3329_reps_0"), val = tensor<int32, [5]>([1, 1, 1, 2, 1])];
+            tensor<fp16, [1, 1024, 16, 2, 128]> var_3329_cast_fp16 = tile(reps = var_3329_reps_0, x = var_3327_cast_fp16)[name = string("op_3329_cast_fp16")];
+            tensor<int32, [4]> var_3330 = const()[name = string("op_3330"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<fp16, [1, 1024, 32, 128]> q_49_cast_fp16 = reshape(shape = var_3330, x = var_3329_cast_fp16)[name = string("q_49_cast_fp16")];
+            tensor<int32, [1]> var_3335_axes_0 = const()[name = string("op_3335_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [1, 1024, 16, 1, 128]> var_3335_cast_fp16 = expand_dims(axes = var_3335_axes_0, x = k_41_cast_fp16)[name = string("op_3335_cast_fp16")];
+            tensor<int32, [5]> var_3337_reps_0 = const()[name = string("op_3337_reps_0"), val = tensor<int32, [5]>([1, 1, 1, 2, 1])];
+            tensor<fp16, [1, 1024, 16, 2, 128]> var_3337_cast_fp16 = tile(reps = var_3337_reps_0, x = var_3335_cast_fp16)[name = string("op_3337_cast_fp16")];
+            tensor<int32, [4]> var_3338 = const()[name = string("op_3338"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<fp16, [1, 1024, 32, 128]> k_43_cast_fp16 = reshape(shape = var_3338, x = var_3337_cast_fp16)[name = string("k_43_cast_fp16")];
+            fp16 var_3340_to_fp16 = const()[name = string("op_3340_to_fp16"), val = fp16(0x1.6ap-4)];
+            tensor<fp16, [1, 1024, 32, 128]> q_51_cast_fp16 = mul(x = q_49_cast_fp16, y = var_3340_to_fp16)[name = string("q_51_cast_fp16")];
+            tensor<int32, [4]> var_3342_begin_0 = const()[name = string("op_3342_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_3342_end_0 = const()[name = string("op_3342_end_0"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<bool, [4]> var_3342_end_mask_0 = const()[name = string("op_3342_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_3342_squeeze_mask_0 = const()[name = string("op_3342_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [1024, 32, 128]> var_3342_cast_fp16 = slice_by_index(begin = var_3342_begin_0, end = var_3342_end_0, end_mask = var_3342_end_mask_0, squeeze_mask = var_3342_squeeze_mask_0, x = q_51_cast_fp16)[name = string("op_3342_cast_fp16")];
+            tensor<int32, [3]> var_3343 = const()[name = string("op_3343"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_3345 = const()[name = string("op_3345"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<fp16, [32, 1024, 128]> var_3344_cast_fp16 = transpose(perm = var_3343, x = var_3342_cast_fp16)[name = string("transpose_17")];
+            tensor<fp16, [32, 16, 64, 128]> q_53_cast_fp16 = reshape(shape = var_3345, x = var_3344_cast_fp16)[name = string("q_53_cast_fp16")];
+            tensor<int32, [4]> var_3347_begin_0 = const()[name = string("op_3347_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_3347_end_0 = const()[name = string("op_3347_end_0"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<bool, [4]> var_3347_end_mask_0 = const()[name = string("op_3347_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_3347_squeeze_mask_0 = const()[name = string("op_3347_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [1024, 32, 128]> var_3347_cast_fp16 = slice_by_index(begin = var_3347_begin_0, end = var_3347_end_0, end_mask = var_3347_end_mask_0, squeeze_mask = var_3347_squeeze_mask_0, x = k_43_cast_fp16)[name = string("op_3347_cast_fp16")];
+            tensor<int32, [3]> var_3348 = const()[name = string("op_3348"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_3350 = const()[name = string("op_3350"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<fp16, [32, 1024, 128]> var_3349_cast_fp16 = transpose(perm = var_3348, x = var_3347_cast_fp16)[name = string("transpose_16")];
+            tensor<fp16, [32, 16, 64, 128]> k_45_cast_fp16 = reshape(shape = var_3350, x = var_3349_cast_fp16)[name = string("k_45_cast_fp16")];
+            tensor<int32, [4]> var_3352_begin_0 = const()[name = string("op_3352_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_3352_end_0 = const()[name = string("op_3352_end_0"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<bool, [4]> var_3352_end_mask_0 = const()[name = string("op_3352_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_3352_squeeze_mask_0 = const()[name = string("op_3352_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [1024, 32, 128]> var_3352_cast_fp16 = slice_by_index(begin = var_3352_begin_0, end = var_3352_end_0, end_mask = var_3352_end_mask_0, squeeze_mask = var_3352_squeeze_mask_0, x = v_21_cast_fp16)[name = string("op_3352_cast_fp16")];
+            tensor<int32, [3]> var_3353 = const()[name = string("op_3353"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_3355 = const()[name = string("op_3355"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<fp16, [32, 1024, 128]> var_3354_cast_fp16 = transpose(perm = var_3353, x = var_3352_cast_fp16)[name = string("transpose_15")];
+            tensor<fp16, [32, 16, 64, 128]> v_23_cast_fp16 = reshape(shape = var_3355, x = var_3354_cast_fp16)[name = string("v_23_cast_fp16")];
+            tensor<int32, [3]> var_3357_begin_0 = const()[name = string("op_3357_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_3357_end_0 = const()[name = string("op_3357_end_0"), val = tensor<int32, [3]>([1, 1024, 32])];
+            tensor<bool, [3]> var_3357_end_mask_0 = const()[name = string("op_3357_end_mask_0"), val = tensor<bool, [3]>([false, true, true])];
+            tensor<bool, [3]> var_3357_squeeze_mask_0 = const()[name = string("op_3357_squeeze_mask_0"), val = tensor<bool, [3]>([true, false, false])];
+            tensor<fp16, [1024, 32]> var_3357_cast_fp16 = slice_by_index(begin = var_3357_begin_0, end = var_3357_end_0, end_mask = var_3357_end_mask_0, squeeze_mask = var_3357_squeeze_mask_0, x = beta_17_cast_fp16)[name = string("op_3357_cast_fp16")];
+            tensor<int32, [2]> var_3358_perm_0 = const()[name = string("op_3358_perm_0"), val = tensor<int32, [2]>([1, 0])];
+            tensor<int32, [4]> var_3359 = const()[name = string("op_3359"), val = tensor<int32, [4]>([32, 16, 64, 1])];
+            tensor<fp16, [32, 1024]> var_3358_cast_fp16 = transpose(perm = var_3358_perm_0, x = var_3357_cast_fp16)[name = string("transpose_14")];
+            tensor<fp16, [32, 16, 64, 1]> beta_19_cast_fp16 = reshape(shape = var_3359, x = var_3358_cast_fp16)[name = string("beta_19_cast_fp16")];
+            tensor<int32, [3]> var_3361_begin_0 = const()[name = string("op_3361_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_3361_end_0 = const()[name = string("op_3361_end_0"), val = tensor<int32, [3]>([1, 1024, 32])];
+            tensor<bool, [3]> var_3361_end_mask_0 = const()[name = string("op_3361_end_mask_0"), val = tensor<bool, [3]>([false, true, true])];
+            tensor<bool, [3]> var_3361_squeeze_mask_0 = const()[name = string("op_3361_squeeze_mask_0"), val = tensor<bool, [3]>([true, false, false])];
+            tensor<fp16, [1024, 32]> var_3361_cast_fp16 = slice_by_index(begin = var_3361_begin_0, end = var_3361_end_0, end_mask = var_3361_end_mask_0, squeeze_mask = var_3361_squeeze_mask_0, x = g_17_cast_fp16)[name = string("op_3361_cast_fp16")];
+            tensor<int32, [2]> var_3362_perm_0 = const()[name = string("op_3362_perm_0"), val = tensor<int32, [2]>([1, 0])];
+            tensor<int32, [3]> var_3363 = const()[name = string("op_3363"), val = tensor<int32, [3]>([32, 16, 64])];
+            tensor<fp16, [32, 1024]> var_3362_cast_fp16 = transpose(perm = var_3362_perm_0, x = var_3361_cast_fp16)[name = string("transpose_13")];
+            tensor<fp16, [32, 16, 64]> g_19_cast_fp16 = reshape(shape = var_3363, x = var_3362_cast_fp16)[name = string("g_19_cast_fp16")];
+            tensor<int32, [1]> g_col_9_axes_0 = const()[name = string("g_col_9_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 16, 64, 1]> g_col_9_cast_fp16 = expand_dims(axes = g_col_9_axes_0, x = g_19_cast_fp16)[name = string("g_col_9_cast_fp16")];
+            bool var_3366_transpose_x_0 = const()[name = string("op_3366_transpose_x_0"), val = bool(false)];
+            bool var_3366_transpose_y_0 = const()[name = string("op_3366_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 16, 64, 1]> var_3366_cast_fp16 = matmul(transpose_x = var_3366_transpose_x_0, transpose_y = var_3366_transpose_y_0, x = layers_0_linear_attn_lower_incl_to_fp16, y = g_col_9_cast_fp16)[name = string("op_3366_cast_fp16")];
+            tensor<int32, [1]> cum_9_axes_0 = const()[name = string("cum_9_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 16, 64]> cum_9_cast_fp16 = squeeze(axes = cum_9_axes_0, x = var_3366_cast_fp16)[name = string("cum_9_cast_fp16")];
+            bool var_3368_transpose_x_0 = const()[name = string("op_3368_transpose_x_0"), val = bool(false)];
+            bool var_3368_transpose_y_0 = const()[name = string("op_3368_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 16, 64, 1]> var_3368_cast_fp16 = matmul(transpose_x = var_3368_transpose_x_0, transpose_y = var_3368_transpose_y_0, x = layers_0_linear_attn_suffix_sum_t_to_fp16, y = g_col_9_cast_fp16)[name = string("op_3368_cast_fp16")];
+            tensor<int32, [1]> to_end_9_axes_0 = const()[name = string("to_end_9_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 16, 64]> to_end_9_cast_fp16 = squeeze(axes = to_end_9_axes_0, x = var_3368_cast_fp16)[name = string("to_end_9_cast_fp16")];
+            bool var_3370_transpose_x_0 = const()[name = string("op_3370_transpose_x_0"), val = bool(false)];
+            bool var_3370_transpose_y_0 = const()[name = string("op_3370_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 16, 4096, 1]> var_3370_cast_fp16 = matmul(transpose_x = var_3370_transpose_x_0, transpose_y = var_3370_transpose_y_0, x = layers_0_linear_attn_pair_sum_t_to_fp16, y = g_col_9_cast_fp16)[name = string("op_3370_cast_fp16")];
+            tensor<int32, [4]> var_3371 = const()[name = string("op_3371"), val = tensor<int32, [4]>([32, 16, 64, 64])];
+            tensor<fp16, [32, 16, 64, 64]> pair_57_cast_fp16 = reshape(shape = var_3371, x = var_3370_cast_fp16)[name = string("pair_57_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 64]> var_3373_cast_fp16 = exp(x = pair_57_cast_fp16)[name = string("op_3373_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 64]> pair_decay_9_cast_fp16 = mul(x = var_3373_cast_fp16, y = layers_0_linear_attn_lower_incl_to_fp16)[name = string("pair_decay_9_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 128]> k_beta_9_cast_fp16 = mul(x = k_45_cast_fp16, y = beta_19_cast_fp16)[name = string("k_beta_9_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 128]> v_beta_9_cast_fp16 = mul(x = v_23_cast_fp16, y = beta_19_cast_fp16)[name = string("v_beta_9_cast_fp16")];
+            bool var_3378_transpose_x_1 = const()[name = string("op_3378_transpose_x_1"), val = bool(false)];
+            bool var_3378_transpose_y_1 = const()[name = string("op_3378_transpose_y_1"), val = bool(true)];
+            tensor<fp16, [32, 16, 64, 64]> var_3378_cast_fp16 = matmul(transpose_x = var_3378_transpose_x_1, transpose_y = var_3378_transpose_y_1, x = k_beta_9_cast_fp16, y = k_45_cast_fp16)[name = string("op_3378_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 64]> kkt_9_cast_fp16 = mul(x = var_3378_cast_fp16, y = pair_decay_9_cast_fp16)[name = string("kkt_9_cast_fp16")];
+            bool var_3381_transpose_x_1 = const()[name = string("op_3381_transpose_x_1"), val = bool(false)];
+            bool var_3381_transpose_y_1 = const()[name = string("op_3381_transpose_y_1"), val = bool(true)];
+            tensor<fp16, [32, 16, 64, 64]> var_3381_cast_fp16 = matmul(transpose_x = var_3381_transpose_x_1, transpose_y = var_3381_transpose_y_1, x = q_53_cast_fp16, y = k_45_cast_fp16)[name = string("op_3381_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 64]> attn_intra_9_cast_fp16 = mul(x = var_3381_cast_fp16, y = pair_decay_9_cast_fp16)[name = string("attn_intra_9_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 64]> var_3383_cast_fp16 = mul(x = kkt_9_cast_fp16, y = layers_0_linear_attn_strict_lower_to_fp16)[name = string("op_3383_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 64]> t_17_cast_fp16 = add(x = layers_0_linear_attn_eye_to_fp16, y = var_3383_cast_fp16)[name = string("t_17_cast_fp16")];
+            tensor<int32, [3]> var_3387 = const()[name = string("op_3387"), val = tensor<int32, [3]>([-1, 64, 64])];
+            tensor<fp16, [512, 64, 64]> t_19_cast_fp16 = reshape(shape = var_3387, x = t_17_cast_fp16)[name = string("t_19_cast_fp16")];
+            tensor<int32, [5]> var_3392 = const()[name = string("op_3392"), val = tensor<int32, [5]>([512, 32, 2, 32, 2])];
+            tensor<fp16, [512, 32, 2, 32, 2]> var_3393_cast_fp16 = reshape(shape = var_3392, x = t_19_cast_fp16)[name = string("op_3393_cast_fp16")];
+            tensor<fp16, [512, 32, 2, 32, 2]> var_3398_cast_fp16 = mul(x = var_3393_cast_fp16, y = var_213_to_fp16)[name = string("op_3398_cast_fp16")];
+            tensor<int32, [1]> blocks_49_axes_0 = const()[name = string("blocks_49_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_49_keep_dims_0 = const()[name = string("blocks_49_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 32, 2, 2]> blocks_49_cast_fp16 = reduce_sum(axes = blocks_49_axes_0, keep_dims = blocks_49_keep_dims_0, x = var_3398_cast_fp16)[name = string("blocks_49_cast_fp16")];
+            tensor<int32, [4]> var_3401_begin_0 = const()[name = string("op_3401_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
+            tensor<int32, [4]> var_3401_end_0 = const()[name = string("op_3401_end_0"), val = tensor<int32, [4]>([512, 32, 2, 2])];
+            tensor<bool, [4]> var_3401_end_mask_0 = const()[name = string("op_3401_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 32, 1, 2]> var_3401_cast_fp16 = slice_by_index(begin = var_3401_begin_0, end = var_3401_end_0, end_mask = var_3401_end_mask_0, x = blocks_49_cast_fp16)[name = string("op_3401_cast_fp16")];
+            tensor<int32, [4]> x_131_begin_0 = const()[name = string("x_131_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_131_end_0 = const()[name = string("x_131_end_0"), val = tensor<int32, [4]>([512, 32, 1, 1])];
+            tensor<bool, [4]> x_131_end_mask_0 = const()[name = string("x_131_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 32, 1, 1]> x_131_cast_fp16 = slice_by_index(begin = x_131_begin_0, end = x_131_end_0, end_mask = x_131_end_mask_0, x = var_3401_cast_fp16)[name = string("x_131_cast_fp16")];
+            bool var_3411_transpose_x_0 = const()[name = string("op_3411_transpose_x_0"), val = bool(false)];
+            bool var_3411_transpose_y_0 = const()[name = string("op_3411_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 32, 1, 1]> var_3411_cast_fp16 = matmul(transpose_x = var_3411_transpose_x_0, transpose_y = var_3411_transpose_y_0, x = x_131_cast_fp16, y = var_221_to_fp16)[name = string("op_3411_cast_fp16")];
+            bool var_3412_transpose_x_0 = const()[name = string("op_3412_transpose_x_0"), val = bool(false)];
+            bool var_3412_transpose_y_0 = const()[name = string("op_3412_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 32, 1, 1]> var_3412_cast_fp16 = matmul(transpose_x = var_3412_transpose_x_0, transpose_y = var_3412_transpose_y_0, x = var_221_to_fp16, y = var_3411_cast_fp16)[name = string("op_3412_cast_fp16")];
+            fp16 const_59_promoted_to_fp16 = const()[name = string("const_59_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 32, 1, 1]> lower_49_cast_fp16 = mul(x = var_3412_cast_fp16, y = const_59_promoted_to_fp16)[name = string("lower_49_cast_fp16")];
+            bool bottom_49_interleave_0 = const()[name = string("bottom_49_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 32, 1, 2]> bottom_49_cast_fp16 = concat(axis = var_3233, interleave = bottom_49_interleave_0, values = (lower_49_cast_fp16, var_221_to_fp16))[name = string("bottom_49_cast_fp16")];
+            bool inv_67_interleave_0 = const()[name = string("inv_67_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 32, 2, 2]> inv_67_cast_fp16 = concat(axis = var_3218, interleave = inv_67_interleave_0, values = (top_1_to_fp16, bottom_49_cast_fp16))[name = string("inv_67_cast_fp16")];
+            tensor<int32, [5]> var_3421 = const()[name = string("op_3421"), val = tensor<int32, [5]>([512, 16, 4, 16, 4])];
+            tensor<fp16, [512, 16, 4, 16, 4]> var_3422_cast_fp16 = reshape(shape = var_3421, x = t_19_cast_fp16)[name = string("op_3422_cast_fp16")];
+            tensor<fp16, [512, 16, 4, 16, 4]> var_3427_cast_fp16 = mul(x = var_3422_cast_fp16, y = var_242_to_fp16)[name = string("op_3427_cast_fp16")];
+            tensor<int32, [1]> blocks_51_axes_0 = const()[name = string("blocks_51_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_51_keep_dims_0 = const()[name = string("blocks_51_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 4, 4]> blocks_51_cast_fp16 = reduce_sum(axes = blocks_51_axes_0, keep_dims = blocks_51_keep_dims_0, x = var_3427_cast_fp16)[name = string("blocks_51_cast_fp16")];
+            tensor<int32, [4]> var_3430_begin_0 = const()[name = string("op_3430_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
+            tensor<int32, [4]> var_3430_end_0 = const()[name = string("op_3430_end_0"), val = tensor<int32, [4]>([512, 16, 4, 4])];
+            tensor<bool, [4]> var_3430_end_mask_0 = const()[name = string("op_3430_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 16, 2, 4]> var_3430_cast_fp16 = slice_by_index(begin = var_3430_begin_0, end = var_3430_end_0, end_mask = var_3430_end_mask_0, x = blocks_51_cast_fp16)[name = string("op_3430_cast_fp16")];
+            tensor<int32, [4]> x_133_begin_0 = const()[name = string("x_133_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_133_end_0 = const()[name = string("x_133_end_0"), val = tensor<int32, [4]>([512, 16, 2, 2])];
+            tensor<bool, [4]> x_133_end_mask_0 = const()[name = string("x_133_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 16, 2, 2]> x_133_cast_fp16 = slice_by_index(begin = x_133_begin_0, end = x_133_end_0, end_mask = x_133_end_mask_0, x = var_3430_cast_fp16)[name = string("x_133_cast_fp16")];
+            tensor<int32, [5]> var_3432 = const()[name = string("op_3432"), val = tensor<int32, [5]>([512, 16, 2, 2, 2])];
+            tensor<fp16, [512, 16, 2, 2, 2]> pair_61_cast_fp16 = reshape(shape = var_3432, x = inv_67_cast_fp16)[name = string("pair_61_cast_fp16")];
+            tensor<int32, [5]> var_3434_begin_0 = const()[name = string("op_3434_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_3434_end_0 = const()[name = string("op_3434_end_0"), val = tensor<int32, [5]>([512, 16, 1, 2, 2])];
+            tensor<bool, [5]> var_3434_end_mask_0 = const()[name = string("op_3434_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_3434_squeeze_mask_0 = const()[name = string("op_3434_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 16, 2, 2]> var_3434_cast_fp16 = slice_by_index(begin = var_3434_begin_0, end = var_3434_end_0, end_mask = var_3434_end_mask_0, squeeze_mask = var_3434_squeeze_mask_0, x = pair_61_cast_fp16)[name = string("op_3434_cast_fp16")];
+            tensor<int32, [5]> var_3437_begin_0 = const()[name = string("op_3437_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_3437_end_0 = const()[name = string("op_3437_end_0"), val = tensor<int32, [5]>([512, 16, 2, 2, 2])];
+            tensor<bool, [5]> var_3437_end_mask_0 = const()[name = string("op_3437_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_3437_squeeze_mask_0 = const()[name = string("op_3437_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 16, 2, 2]> var_3437_cast_fp16 = slice_by_index(begin = var_3437_begin_0, end = var_3437_end_0, end_mask = var_3437_end_mask_0, squeeze_mask = var_3437_squeeze_mask_0, x = pair_61_cast_fp16)[name = string("op_3437_cast_fp16")];
+            bool var_3440_transpose_x_0 = const()[name = string("op_3440_transpose_x_0"), val = bool(false)];
+            bool var_3440_transpose_y_0 = const()[name = string("op_3440_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 2, 2]> var_3440_cast_fp16 = matmul(transpose_x = var_3440_transpose_x_0, transpose_y = var_3440_transpose_y_0, x = x_133_cast_fp16, y = var_3434_cast_fp16)[name = string("op_3440_cast_fp16")];
+            bool var_3441_transpose_x_0 = const()[name = string("op_3441_transpose_x_0"), val = bool(false)];
+            bool var_3441_transpose_y_0 = const()[name = string("op_3441_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 2, 2]> var_3441_cast_fp16 = matmul(transpose_x = var_3441_transpose_x_0, transpose_y = var_3441_transpose_y_0, x = var_3437_cast_fp16, y = var_3440_cast_fp16)[name = string("op_3441_cast_fp16")];
+            fp16 const_60_promoted_to_fp16 = const()[name = string("const_60_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 16, 2, 2]> lower_51_cast_fp16 = mul(x = var_3441_cast_fp16, y = const_60_promoted_to_fp16)[name = string("lower_51_cast_fp16")];
+            bool top_51_interleave_0 = const()[name = string("top_51_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 2, 4]> top_51_cast_fp16 = concat(axis = var_3233, interleave = top_51_interleave_0, values = (var_3434_cast_fp16, var_259_to_fp16))[name = string("top_51_cast_fp16")];
+            bool bottom_51_interleave_0 = const()[name = string("bottom_51_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 2, 4]> bottom_51_cast_fp16 = concat(axis = var_3233, interleave = bottom_51_interleave_0, values = (lower_51_cast_fp16, var_3437_cast_fp16))[name = string("bottom_51_cast_fp16")];
+            bool inv_69_interleave_0 = const()[name = string("inv_69_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 4, 4]> inv_69_cast_fp16 = concat(axis = var_3218, interleave = inv_69_interleave_0, values = (top_51_cast_fp16, bottom_51_cast_fp16))[name = string("inv_69_cast_fp16")];
+            tensor<int32, [5]> var_3450 = const()[name = string("op_3450"), val = tensor<int32, [5]>([512, 8, 8, 8, 8])];
+            tensor<fp16, [512, 8, 8, 8, 8]> var_3451_cast_fp16 = reshape(shape = var_3450, x = t_19_cast_fp16)[name = string("op_3451_cast_fp16")];
+            tensor<fp16, [8, 1, 8, 1]> var_3455_to_fp16 = const()[name = string("op_3455_to_fp16"), val = tensor<fp16, [8, 1, 8, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(687944576)))];
+            tensor<fp16, [512, 8, 8, 8, 8]> var_3456_cast_fp16 = mul(x = var_3451_cast_fp16, y = var_3455_to_fp16)[name = string("op_3456_cast_fp16")];
+            tensor<int32, [1]> blocks_53_axes_0 = const()[name = string("blocks_53_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_53_keep_dims_0 = const()[name = string("blocks_53_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 8, 8]> blocks_53_cast_fp16 = reduce_sum(axes = blocks_53_axes_0, keep_dims = blocks_53_keep_dims_0, x = var_3456_cast_fp16)[name = string("blocks_53_cast_fp16")];
+            tensor<int32, [4]> var_3459_begin_0 = const()[name = string("op_3459_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
+            tensor<int32, [4]> var_3459_end_0 = const()[name = string("op_3459_end_0"), val = tensor<int32, [4]>([512, 8, 8, 8])];
+            tensor<bool, [4]> var_3459_end_mask_0 = const()[name = string("op_3459_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 8, 4, 8]> var_3459_cast_fp16 = slice_by_index(begin = var_3459_begin_0, end = var_3459_end_0, end_mask = var_3459_end_mask_0, x = blocks_53_cast_fp16)[name = string("op_3459_cast_fp16")];
+            tensor<int32, [4]> x_135_begin_0 = const()[name = string("x_135_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_135_end_0 = const()[name = string("x_135_end_0"), val = tensor<int32, [4]>([512, 8, 4, 4])];
+            tensor<bool, [4]> x_135_end_mask_0 = const()[name = string("x_135_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 8, 4, 4]> x_135_cast_fp16 = slice_by_index(begin = x_135_begin_0, end = x_135_end_0, end_mask = x_135_end_mask_0, x = var_3459_cast_fp16)[name = string("x_135_cast_fp16")];
+            tensor<int32, [5]> var_3461 = const()[name = string("op_3461"), val = tensor<int32, [5]>([512, 8, 2, 4, 4])];
+            tensor<fp16, [512, 8, 2, 4, 4]> pair_63_cast_fp16 = reshape(shape = var_3461, x = inv_69_cast_fp16)[name = string("pair_63_cast_fp16")];
+            tensor<int32, [5]> var_3463_begin_0 = const()[name = string("op_3463_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_3463_end_0 = const()[name = string("op_3463_end_0"), val = tensor<int32, [5]>([512, 8, 1, 4, 4])];
+            tensor<bool, [5]> var_3463_end_mask_0 = const()[name = string("op_3463_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_3463_squeeze_mask_0 = const()[name = string("op_3463_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 8, 4, 4]> var_3463_cast_fp16 = slice_by_index(begin = var_3463_begin_0, end = var_3463_end_0, end_mask = var_3463_end_mask_0, squeeze_mask = var_3463_squeeze_mask_0, x = pair_63_cast_fp16)[name = string("op_3463_cast_fp16")];
+            tensor<int32, [5]> var_3466_begin_0 = const()[name = string("op_3466_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_3466_end_0 = const()[name = string("op_3466_end_0"), val = tensor<int32, [5]>([512, 8, 2, 4, 4])];
+            tensor<bool, [5]> var_3466_end_mask_0 = const()[name = string("op_3466_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_3466_squeeze_mask_0 = const()[name = string("op_3466_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 8, 4, 4]> var_3466_cast_fp16 = slice_by_index(begin = var_3466_begin_0, end = var_3466_end_0, end_mask = var_3466_end_mask_0, squeeze_mask = var_3466_squeeze_mask_0, x = pair_63_cast_fp16)[name = string("op_3466_cast_fp16")];
+            bool var_3469_transpose_x_0 = const()[name = string("op_3469_transpose_x_0"), val = bool(false)];
+            bool var_3469_transpose_y_0 = const()[name = string("op_3469_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 4, 4]> var_3469_cast_fp16 = matmul(transpose_x = var_3469_transpose_x_0, transpose_y = var_3469_transpose_y_0, x = x_135_cast_fp16, y = var_3463_cast_fp16)[name = string("op_3469_cast_fp16")];
+            bool var_3470_transpose_x_0 = const()[name = string("op_3470_transpose_x_0"), val = bool(false)];
+            bool var_3470_transpose_y_0 = const()[name = string("op_3470_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 4, 4]> var_3470_cast_fp16 = matmul(transpose_x = var_3470_transpose_x_0, transpose_y = var_3470_transpose_y_0, x = var_3466_cast_fp16, y = var_3469_cast_fp16)[name = string("op_3470_cast_fp16")];
+            fp16 const_61_promoted_to_fp16 = const()[name = string("const_61_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 8, 4, 4]> lower_53_cast_fp16 = mul(x = var_3470_cast_fp16, y = const_61_promoted_to_fp16)[name = string("lower_53_cast_fp16")];
+            bool top_53_interleave_0 = const()[name = string("top_53_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 4, 8]> top_53_cast_fp16 = concat(axis = var_3233, interleave = top_53_interleave_0, values = (var_3463_cast_fp16, var_288_to_fp16))[name = string("top_53_cast_fp16")];
+            bool bottom_53_interleave_0 = const()[name = string("bottom_53_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 4, 8]> bottom_53_cast_fp16 = concat(axis = var_3233, interleave = bottom_53_interleave_0, values = (lower_53_cast_fp16, var_3466_cast_fp16))[name = string("bottom_53_cast_fp16")];
+            bool inv_71_interleave_0 = const()[name = string("inv_71_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 8, 8]> inv_71_cast_fp16 = concat(axis = var_3218, interleave = inv_71_interleave_0, values = (top_53_cast_fp16, bottom_53_cast_fp16))[name = string("inv_71_cast_fp16")];
+            tensor<int32, [5]> var_3479 = const()[name = string("op_3479"), val = tensor<int32, [5]>([512, 4, 16, 4, 16])];
+            tensor<fp16, [512, 4, 16, 4, 16]> var_3480_cast_fp16 = reshape(shape = var_3479, x = t_19_cast_fp16)[name = string("op_3480_cast_fp16")];
+            tensor<fp16, [4, 1, 4, 1]> var_3484_to_fp16 = const()[name = string("op_3484_to_fp16"), val = tensor<fp16, [4, 1, 4, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(687944768)))];
+            tensor<fp16, [512, 4, 16, 4, 16]> var_3485_cast_fp16 = mul(x = var_3480_cast_fp16, y = var_3484_to_fp16)[name = string("op_3485_cast_fp16")];
+            tensor<int32, [1]> blocks_55_axes_0 = const()[name = string("blocks_55_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_55_keep_dims_0 = const()[name = string("blocks_55_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 16, 16]> blocks_55_cast_fp16 = reduce_sum(axes = blocks_55_axes_0, keep_dims = blocks_55_keep_dims_0, x = var_3485_cast_fp16)[name = string("blocks_55_cast_fp16")];
+            tensor<int32, [4]> var_3488_begin_0 = const()[name = string("op_3488_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
+            tensor<int32, [4]> var_3488_end_0 = const()[name = string("op_3488_end_0"), val = tensor<int32, [4]>([512, 4, 16, 16])];
+            tensor<bool, [4]> var_3488_end_mask_0 = const()[name = string("op_3488_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 4, 8, 16]> var_3488_cast_fp16 = slice_by_index(begin = var_3488_begin_0, end = var_3488_end_0, end_mask = var_3488_end_mask_0, x = blocks_55_cast_fp16)[name = string("op_3488_cast_fp16")];
+            tensor<int32, [4]> x_137_begin_0 = const()[name = string("x_137_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_137_end_0 = const()[name = string("x_137_end_0"), val = tensor<int32, [4]>([512, 4, 8, 8])];
+            tensor<bool, [4]> x_137_end_mask_0 = const()[name = string("x_137_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 4, 8, 8]> x_137_cast_fp16 = slice_by_index(begin = x_137_begin_0, end = x_137_end_0, end_mask = x_137_end_mask_0, x = var_3488_cast_fp16)[name = string("x_137_cast_fp16")];
+            tensor<int32, [5]> var_3490 = const()[name = string("op_3490"), val = tensor<int32, [5]>([512, 4, 2, 8, 8])];
+            tensor<fp16, [512, 4, 2, 8, 8]> pair_65_cast_fp16 = reshape(shape = var_3490, x = inv_71_cast_fp16)[name = string("pair_65_cast_fp16")];
+            tensor<int32, [5]> var_3492_begin_0 = const()[name = string("op_3492_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_3492_end_0 = const()[name = string("op_3492_end_0"), val = tensor<int32, [5]>([512, 4, 1, 8, 8])];
+            tensor<bool, [5]> var_3492_end_mask_0 = const()[name = string("op_3492_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_3492_squeeze_mask_0 = const()[name = string("op_3492_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 4, 8, 8]> var_3492_cast_fp16 = slice_by_index(begin = var_3492_begin_0, end = var_3492_end_0, end_mask = var_3492_end_mask_0, squeeze_mask = var_3492_squeeze_mask_0, x = pair_65_cast_fp16)[name = string("op_3492_cast_fp16")];
+            tensor<int32, [5]> var_3495_begin_0 = const()[name = string("op_3495_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_3495_end_0 = const()[name = string("op_3495_end_0"), val = tensor<int32, [5]>([512, 4, 2, 8, 8])];
+            tensor<bool, [5]> var_3495_end_mask_0 = const()[name = string("op_3495_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_3495_squeeze_mask_0 = const()[name = string("op_3495_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 4, 8, 8]> var_3495_cast_fp16 = slice_by_index(begin = var_3495_begin_0, end = var_3495_end_0, end_mask = var_3495_end_mask_0, squeeze_mask = var_3495_squeeze_mask_0, x = pair_65_cast_fp16)[name = string("op_3495_cast_fp16")];
+            bool var_3498_transpose_x_0 = const()[name = string("op_3498_transpose_x_0"), val = bool(false)];
+            bool var_3498_transpose_y_0 = const()[name = string("op_3498_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 8, 8]> var_3498_cast_fp16 = matmul(transpose_x = var_3498_transpose_x_0, transpose_y = var_3498_transpose_y_0, x = x_137_cast_fp16, y = var_3492_cast_fp16)[name = string("op_3498_cast_fp16")];
+            bool var_3499_transpose_x_0 = const()[name = string("op_3499_transpose_x_0"), val = bool(false)];
+            bool var_3499_transpose_y_0 = const()[name = string("op_3499_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 8, 8]> var_3499_cast_fp16 = matmul(transpose_x = var_3499_transpose_x_0, transpose_y = var_3499_transpose_y_0, x = var_3495_cast_fp16, y = var_3498_cast_fp16)[name = string("op_3499_cast_fp16")];
+            fp16 const_62_promoted_to_fp16 = const()[name = string("const_62_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 4, 8, 8]> lower_55_cast_fp16 = mul(x = var_3499_cast_fp16, y = const_62_promoted_to_fp16)[name = string("lower_55_cast_fp16")];
+            bool top_55_interleave_0 = const()[name = string("top_55_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 8, 16]> top_55_cast_fp16 = concat(axis = var_3233, interleave = top_55_interleave_0, values = (var_3492_cast_fp16, var_317_to_fp16))[name = string("top_55_cast_fp16")];
+            bool bottom_55_interleave_0 = const()[name = string("bottom_55_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 8, 16]> bottom_55_cast_fp16 = concat(axis = var_3233, interleave = bottom_55_interleave_0, values = (lower_55_cast_fp16, var_3495_cast_fp16))[name = string("bottom_55_cast_fp16")];
+            bool inv_73_interleave_0 = const()[name = string("inv_73_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 16, 16]> inv_73_cast_fp16 = concat(axis = var_3218, interleave = inv_73_interleave_0, values = (top_55_cast_fp16, bottom_55_cast_fp16))[name = string("inv_73_cast_fp16")];
+            tensor<int32, [5]> var_3508 = const()[name = string("op_3508"), val = tensor<int32, [5]>([512, 2, 32, 2, 32])];
+            tensor<fp16, [512, 2, 32, 2, 32]> var_3509_cast_fp16 = reshape(shape = var_3508, x = t_19_cast_fp16)[name = string("op_3509_cast_fp16")];
+            tensor<fp16, [2, 1, 2, 1]> var_3513_to_fp16 = const()[name = string("op_3513_to_fp16"), val = tensor<fp16, [2, 1, 2, 1]>([[[[0x1p+0], [0x0p+0]]], [[[0x0p+0], [0x1p+0]]]])];
+            tensor<fp16, [512, 2, 32, 2, 32]> var_3514_cast_fp16 = mul(x = var_3509_cast_fp16, y = var_3513_to_fp16)[name = string("op_3514_cast_fp16")];
+            tensor<int32, [1]> blocks_57_axes_0 = const()[name = string("blocks_57_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_57_keep_dims_0 = const()[name = string("blocks_57_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 32, 32]> blocks_57_cast_fp16 = reduce_sum(axes = blocks_57_axes_0, keep_dims = blocks_57_keep_dims_0, x = var_3514_cast_fp16)[name = string("blocks_57_cast_fp16")];
+            tensor<int32, [4]> var_3517_begin_0 = const()[name = string("op_3517_begin_0"), val = tensor<int32, [4]>([0, 0, 16, 0])];
+            tensor<int32, [4]> var_3517_end_0 = const()[name = string("op_3517_end_0"), val = tensor<int32, [4]>([512, 2, 32, 32])];
+            tensor<bool, [4]> var_3517_end_mask_0 = const()[name = string("op_3517_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 2, 16, 32]> var_3517_cast_fp16 = slice_by_index(begin = var_3517_begin_0, end = var_3517_end_0, end_mask = var_3517_end_mask_0, x = blocks_57_cast_fp16)[name = string("op_3517_cast_fp16")];
+            tensor<int32, [4]> x_139_begin_0 = const()[name = string("x_139_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_139_end_0 = const()[name = string("x_139_end_0"), val = tensor<int32, [4]>([512, 2, 16, 16])];
+            tensor<bool, [4]> x_139_end_mask_0 = const()[name = string("x_139_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 2, 16, 16]> x_139_cast_fp16 = slice_by_index(begin = x_139_begin_0, end = x_139_end_0, end_mask = x_139_end_mask_0, x = var_3517_cast_fp16)[name = string("x_139_cast_fp16")];
+            tensor<int32, [5]> var_3519 = const()[name = string("op_3519"), val = tensor<int32, [5]>([512, 2, 2, 16, 16])];
+            tensor<fp16, [512, 2, 2, 16, 16]> pair_67_cast_fp16 = reshape(shape = var_3519, x = inv_73_cast_fp16)[name = string("pair_67_cast_fp16")];
+            tensor<int32, [5]> var_3521_begin_0 = const()[name = string("op_3521_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_3521_end_0 = const()[name = string("op_3521_end_0"), val = tensor<int32, [5]>([512, 2, 1, 16, 16])];
+            tensor<bool, [5]> var_3521_end_mask_0 = const()[name = string("op_3521_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_3521_squeeze_mask_0 = const()[name = string("op_3521_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 2, 16, 16]> var_3521_cast_fp16 = slice_by_index(begin = var_3521_begin_0, end = var_3521_end_0, end_mask = var_3521_end_mask_0, squeeze_mask = var_3521_squeeze_mask_0, x = pair_67_cast_fp16)[name = string("op_3521_cast_fp16")];
+            tensor<int32, [5]> var_3524_begin_0 = const()[name = string("op_3524_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_3524_end_0 = const()[name = string("op_3524_end_0"), val = tensor<int32, [5]>([512, 2, 2, 16, 16])];
+            tensor<bool, [5]> var_3524_end_mask_0 = const()[name = string("op_3524_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_3524_squeeze_mask_0 = const()[name = string("op_3524_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 2, 16, 16]> var_3524_cast_fp16 = slice_by_index(begin = var_3524_begin_0, end = var_3524_end_0, end_mask = var_3524_end_mask_0, squeeze_mask = var_3524_squeeze_mask_0, x = pair_67_cast_fp16)[name = string("op_3524_cast_fp16")];
+            bool var_3527_transpose_x_0 = const()[name = string("op_3527_transpose_x_0"), val = bool(false)];
+            bool var_3527_transpose_y_0 = const()[name = string("op_3527_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 16, 16]> var_3527_cast_fp16 = matmul(transpose_x = var_3527_transpose_x_0, transpose_y = var_3527_transpose_y_0, x = x_139_cast_fp16, y = var_3521_cast_fp16)[name = string("op_3527_cast_fp16")];
+            bool var_3528_transpose_x_0 = const()[name = string("op_3528_transpose_x_0"), val = bool(false)];
+            bool var_3528_transpose_y_0 = const()[name = string("op_3528_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 16, 16]> var_3528_cast_fp16 = matmul(transpose_x = var_3528_transpose_x_0, transpose_y = var_3528_transpose_y_0, x = var_3524_cast_fp16, y = var_3527_cast_fp16)[name = string("op_3528_cast_fp16")];
+            fp16 const_63_promoted_to_fp16 = const()[name = string("const_63_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 2, 16, 16]> lower_57_cast_fp16 = mul(x = var_3528_cast_fp16, y = const_63_promoted_to_fp16)[name = string("lower_57_cast_fp16")];
+            bool top_57_interleave_0 = const()[name = string("top_57_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 16, 32]> top_57_cast_fp16 = concat(axis = var_3233, interleave = top_57_interleave_0, values = (var_3521_cast_fp16, var_346_to_fp16))[name = string("top_57_cast_fp16")];
+            bool bottom_57_interleave_0 = const()[name = string("bottom_57_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 16, 32]> bottom_57_cast_fp16 = concat(axis = var_3233, interleave = bottom_57_interleave_0, values = (lower_57_cast_fp16, var_3524_cast_fp16))[name = string("bottom_57_cast_fp16")];
+            bool inv_75_interleave_0 = const()[name = string("inv_75_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 32, 32]> inv_75_cast_fp16 = concat(axis = var_3218, interleave = inv_75_interleave_0, values = (top_57_cast_fp16, bottom_57_cast_fp16))[name = string("inv_75_cast_fp16")];
+            tensor<int32, [5]> var_3537 = const()[name = string("op_3537"), val = tensor<int32, [5]>([512, 1, 64, 1, 64])];
+            tensor<fp16, [512, 1, 64, 1, 64]> var_3538_cast_fp16 = reshape(shape = var_3537, x = t_19_cast_fp16)[name = string("op_3538_cast_fp16")];
+            tensor<int32, [1]> blocks_59_axes_0 = const()[name = string("blocks_59_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_59_keep_dims_0 = const()[name = string("blocks_59_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 64, 64]> blocks_59_cast_fp16 = reduce_sum(axes = blocks_59_axes_0, keep_dims = blocks_59_keep_dims_0, x = var_3538_cast_fp16)[name = string("blocks_59_cast_fp16")];
+            tensor<int32, [4]> var_3546_begin_0 = const()[name = string("op_3546_begin_0"), val = tensor<int32, [4]>([0, 0, 32, 0])];
+            tensor<int32, [4]> var_3546_end_0 = const()[name = string("op_3546_end_0"), val = tensor<int32, [4]>([512, 1, 64, 64])];
+            tensor<bool, [4]> var_3546_end_mask_0 = const()[name = string("op_3546_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 1, 32, 64]> var_3546_cast_fp16 = slice_by_index(begin = var_3546_begin_0, end = var_3546_end_0, end_mask = var_3546_end_mask_0, x = blocks_59_cast_fp16)[name = string("op_3546_cast_fp16")];
+            tensor<int32, [4]> x_141_begin_0 = const()[name = string("x_141_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_141_end_0 = const()[name = string("x_141_end_0"), val = tensor<int32, [4]>([512, 1, 32, 32])];
+            tensor<bool, [4]> x_141_end_mask_0 = const()[name = string("x_141_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 1, 32, 32]> x_141_cast_fp16 = slice_by_index(begin = x_141_begin_0, end = x_141_end_0, end_mask = x_141_end_mask_0, x = var_3546_cast_fp16)[name = string("x_141_cast_fp16")];
+            tensor<int32, [5]> var_3548 = const()[name = string("op_3548"), val = tensor<int32, [5]>([512, 1, 2, 32, 32])];
+            tensor<fp16, [512, 1, 2, 32, 32]> pair_69_cast_fp16 = reshape(shape = var_3548, x = inv_75_cast_fp16)[name = string("pair_69_cast_fp16")];
+            tensor<int32, [5]> var_3550_begin_0 = const()[name = string("op_3550_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_3550_end_0 = const()[name = string("op_3550_end_0"), val = tensor<int32, [5]>([512, 1, 1, 32, 32])];
+            tensor<bool, [5]> var_3550_end_mask_0 = const()[name = string("op_3550_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_3550_squeeze_mask_0 = const()[name = string("op_3550_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 1, 32, 32]> var_3550_cast_fp16 = slice_by_index(begin = var_3550_begin_0, end = var_3550_end_0, end_mask = var_3550_end_mask_0, squeeze_mask = var_3550_squeeze_mask_0, x = pair_69_cast_fp16)[name = string("op_3550_cast_fp16")];
+            tensor<int32, [5]> var_3553_begin_0 = const()[name = string("op_3553_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_3553_end_0 = const()[name = string("op_3553_end_0"), val = tensor<int32, [5]>([512, 1, 2, 32, 32])];
+            tensor<bool, [5]> var_3553_end_mask_0 = const()[name = string("op_3553_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_3553_squeeze_mask_0 = const()[name = string("op_3553_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 1, 32, 32]> var_3553_cast_fp16 = slice_by_index(begin = var_3553_begin_0, end = var_3553_end_0, end_mask = var_3553_end_mask_0, squeeze_mask = var_3553_squeeze_mask_0, x = pair_69_cast_fp16)[name = string("op_3553_cast_fp16")];
+            bool var_3556_transpose_x_0 = const()[name = string("op_3556_transpose_x_0"), val = bool(false)];
+            bool var_3556_transpose_y_0 = const()[name = string("op_3556_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 32, 32]> var_3556_cast_fp16 = matmul(transpose_x = var_3556_transpose_x_0, transpose_y = var_3556_transpose_y_0, x = x_141_cast_fp16, y = var_3550_cast_fp16)[name = string("op_3556_cast_fp16")];
+            bool var_3557_transpose_x_0 = const()[name = string("op_3557_transpose_x_0"), val = bool(false)];
+            bool var_3557_transpose_y_0 = const()[name = string("op_3557_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 32, 32]> var_3557_cast_fp16 = matmul(transpose_x = var_3557_transpose_x_0, transpose_y = var_3557_transpose_y_0, x = var_3553_cast_fp16, y = var_3556_cast_fp16)[name = string("op_3557_cast_fp16")];
+            fp16 const_64_promoted_to_fp16 = const()[name = string("const_64_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 1, 32, 32]> lower_59_cast_fp16 = mul(x = var_3557_cast_fp16, y = const_64_promoted_to_fp16)[name = string("lower_59_cast_fp16")];
+            bool top_59_interleave_0 = const()[name = string("top_59_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 32, 64]> top_59_cast_fp16 = concat(axis = var_3233, interleave = top_59_interleave_0, values = (var_3550_cast_fp16, var_375_to_fp16))[name = string("top_59_cast_fp16")];
+            bool bottom_59_interleave_0 = const()[name = string("bottom_59_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 32, 64]> bottom_59_cast_fp16 = concat(axis = var_3233, interleave = bottom_59_interleave_0, values = (lower_59_cast_fp16, var_3553_cast_fp16))[name = string("bottom_59_cast_fp16")];
+            bool inv_77_interleave_0 = const()[name = string("inv_77_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 64, 64]> inv_77_cast_fp16 = concat(axis = var_3218, interleave = inv_77_interleave_0, values = (top_59_cast_fp16, bottom_59_cast_fp16))[name = string("inv_77_cast_fp16")];
+            tensor<int32, [4]> var_3566 = const()[name = string("op_3566"), val = tensor<int32, [4]>([32, 16, 64, 64])];
+            tensor<fp16, [32, 16, 64, 64]> inv_79_cast_fp16 = reshape(shape = var_3566, x = inv_77_cast_fp16)[name = string("inv_79_cast_fp16")];
+            bool new_v_9_transpose_x_0 = const()[name = string("new_v_9_transpose_x_0"), val = bool(false)];
+            bool new_v_9_transpose_y_0 = const()[name = string("new_v_9_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 16, 64, 128]> new_v_9_cast_fp16 = matmul(transpose_x = new_v_9_transpose_x_0, transpose_y = new_v_9_transpose_y_0, x = inv_79_cast_fp16, y = v_beta_9_cast_fp16)[name = string("new_v_9_cast_fp16")];
+            tensor<fp16, [32, 16, 64]> var_3569_cast_fp16 = exp(x = cum_9_cast_fp16)[name = string("op_3569_cast_fp16")];
+            tensor<int32, [1]> var_3570_axes_0 = const()[name = string("op_3570_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 16, 64, 1]> var_3570_cast_fp16 = expand_dims(axes = var_3570_axes_0, x = var_3569_cast_fp16)[name = string("op_3570_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 128]> var_3571_cast_fp16 = mul(x = k_beta_9_cast_fp16, y = var_3570_cast_fp16)[name = string("op_3571_cast_fp16")];
+            bool k_cumdecay_9_transpose_x_0 = const()[name = string("k_cumdecay_9_transpose_x_0"), val = bool(false)];
+            bool k_cumdecay_9_transpose_y_0 = const()[name = string("k_cumdecay_9_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 16, 64, 128]> k_cumdecay_9_cast_fp16 = matmul(transpose_x = k_cumdecay_9_transpose_x_0, transpose_y = k_cumdecay_9_transpose_y_0, x = inv_79_cast_fp16, y = var_3571_cast_fp16)[name = string("k_cumdecay_9_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 128]> q_dec_9_cast_fp16 = mul(x = q_53_cast_fp16, y = var_3570_cast_fp16)[name = string("q_dec_9_cast_fp16")];
+            tensor<fp16, [32, 16, 64]> var_3576_cast_fp16 = exp(x = to_end_9_cast_fp16)[name = string("op_3576_cast_fp16")];
+            tensor<int32, [1]> var_3577_axes_0 = const()[name = string("op_3577_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 16, 64, 1]> var_3577_cast_fp16 = expand_dims(axes = var_3577_axes_0, x = var_3576_cast_fp16)[name = string("op_3577_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 128]> k_dec_9_cast_fp16 = mul(x = k_45_cast_fp16, y = var_3577_cast_fp16)[name = string("k_dec_9_cast_fp16")];
+            tensor<int32, [3]> var_3579_begin_0 = const()[name = string("op_3579_begin_0"), val = tensor<int32, [3]>([0, 0, -1])];
+            tensor<int32, [3]> var_3579_end_0 = const()[name = string("op_3579_end_0"), val = tensor<int32, [3]>([32, 16, 64])];
+            tensor<bool, [3]> var_3579_end_mask_0 = const()[name = string("op_3579_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
+            tensor<fp16, [32, 16, 1]> var_3579_cast_fp16 = slice_by_index(begin = var_3579_begin_0, end = var_3579_end_0, end_mask = var_3579_end_mask_0, x = cum_9_cast_fp16)[name = string("op_3579_cast_fp16")];
+            tensor<fp16, [32, 16, 1]> var_3580_cast_fp16 = exp(x = var_3579_cast_fp16)[name = string("op_3580_cast_fp16")];
+            tensor<int32, [1]> chunk_decay_9_axes_0 = const()[name = string("chunk_decay_9_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 16, 1, 1]> chunk_decay_9_cast_fp16 = expand_dims(axes = chunk_decay_9_axes_0, x = var_3580_cast_fp16)[name = string("chunk_decay_9_cast_fp16")];
+            tensor<int32, [4]> var_3585_begin_0 = const()[name = string("op_3585_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_3585_end_0 = const()[name = string("op_3585_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_3585_end_mask_0 = const()[name = string("op_3585_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3585_squeeze_mask_0 = const()[name = string("op_3585_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3585_cast_fp16 = slice_by_index(begin = var_3585_begin_0, end = var_3585_end_0, end_mask = var_3585_end_mask_0, squeeze_mask = var_3585_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_3585_cast_fp16")];
+            tensor<int32, [4]> var_3587_begin_0 = const()[name = string("op_3587_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_3587_end_0 = const()[name = string("op_3587_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_3587_end_mask_0 = const()[name = string("op_3587_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3587_squeeze_mask_0 = const()[name = string("op_3587_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3587_cast_fp16 = slice_by_index(begin = var_3587_begin_0, end = var_3587_end_0, end_mask = var_3587_end_mask_0, squeeze_mask = var_3587_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_3587_cast_fp16")];
+            bool var_3588_transpose_x_0 = const()[name = string("op_3588_transpose_x_0"), val = bool(false)];
+            bool var_3588_transpose_y_0 = const()[name = string("op_3588_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3588_cast_fp16 = matmul(transpose_x = var_3588_transpose_x_0, transpose_y = var_3588_transpose_y_0, x = var_3587_cast_fp16, y = state_1_to_fp16)[name = string("op_3588_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_129_cast_fp16 = sub(x = var_3585_cast_fp16, y = var_3588_cast_fp16)[name = string("v_new_129_cast_fp16")];
+            tensor<int32, [4]> var_3591_begin_0 = const()[name = string("op_3591_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_3591_end_0 = const()[name = string("op_3591_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_3591_end_mask_0 = const()[name = string("op_3591_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3591_squeeze_mask_0 = const()[name = string("op_3591_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3591_cast_fp16 = slice_by_index(begin = var_3591_begin_0, end = var_3591_end_0, end_mask = var_3591_end_mask_0, squeeze_mask = var_3591_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_3591_cast_fp16")];
+            bool var_3592_transpose_x_0 = const()[name = string("op_3592_transpose_x_0"), val = bool(false)];
+            bool var_3592_transpose_y_0 = const()[name = string("op_3592_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3592_cast_fp16 = matmul(transpose_x = var_3592_transpose_x_0, transpose_y = var_3592_transpose_y_0, x = var_3591_cast_fp16, y = state_1_to_fp16)[name = string("op_3592_cast_fp16")];
+            tensor<int32, [4]> var_3594_begin_0 = const()[name = string("op_3594_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_3594_end_0 = const()[name = string("op_3594_end_0"), val = tensor<int32, [4]>([32, 1, 64, 64])];
+            tensor<bool, [4]> var_3594_end_mask_0 = const()[name = string("op_3594_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3594_squeeze_mask_0 = const()[name = string("op_3594_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3594_cast_fp16 = slice_by_index(begin = var_3594_begin_0, end = var_3594_end_0, end_mask = var_3594_end_mask_0, squeeze_mask = var_3594_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_3594_cast_fp16")];
+            bool var_3595_transpose_x_0 = const()[name = string("op_3595_transpose_x_0"), val = bool(false)];
+            bool var_3595_transpose_y_0 = const()[name = string("op_3595_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3595_cast_fp16 = matmul(transpose_x = var_3595_transpose_x_0, transpose_y = var_3595_transpose_y_0, x = var_3594_cast_fp16, y = v_new_129_cast_fp16)[name = string("op_3595_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3596_cast_fp16 = add(x = var_3592_cast_fp16, y = var_3595_cast_fp16)[name = string("op_3596_cast_fp16")];
+            tensor<int32, [4]> var_3598_begin_0 = const()[name = string("op_3598_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_3598_end_0 = const()[name = string("op_3598_end_0"), val = tensor<int32, [4]>([32, 1, 1, 1])];
+            tensor<bool, [4]> var_3598_end_mask_0 = const()[name = string("op_3598_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3598_squeeze_mask_0 = const()[name = string("op_3598_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3598_cast_fp16 = slice_by_index(begin = var_3598_begin_0, end = var_3598_end_0, end_mask = var_3598_end_mask_0, squeeze_mask = var_3598_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_3598_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3599_cast_fp16 = mul(x = state_1_to_fp16, y = var_3598_cast_fp16)[name = string("op_3599_cast_fp16")];
+            tensor<int32, [4]> var_3601_begin_0 = const()[name = string("op_3601_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_3601_end_0 = const()[name = string("op_3601_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_3601_end_mask_0 = const()[name = string("op_3601_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3601_squeeze_mask_0 = const()[name = string("op_3601_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3601_cast_fp16 = slice_by_index(begin = var_3601_begin_0, end = var_3601_end_0, end_mask = var_3601_end_mask_0, squeeze_mask = var_3601_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_3601_cast_fp16")];
+            bool var_3603_transpose_x_1 = const()[name = string("op_3603_transpose_x_1"), val = bool(true)];
+            bool var_3603_transpose_y_1 = const()[name = string("op_3603_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3603_cast_fp16 = matmul(transpose_x = var_3603_transpose_x_1, transpose_y = var_3603_transpose_y_1, x = var_3601_cast_fp16, y = v_new_129_cast_fp16)[name = string("op_3603_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_131_cast_fp16 = add(x = var_3599_cast_fp16, y = var_3603_cast_fp16)[name = string("state_131_cast_fp16")];
+            tensor<int32, [4]> var_3606_begin_0 = const()[name = string("op_3606_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_3606_end_0 = const()[name = string("op_3606_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_3606_end_mask_0 = const()[name = string("op_3606_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3606_squeeze_mask_0 = const()[name = string("op_3606_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3606_cast_fp16 = slice_by_index(begin = var_3606_begin_0, end = var_3606_end_0, end_mask = var_3606_end_mask_0, squeeze_mask = var_3606_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_3606_cast_fp16")];
+            tensor<int32, [4]> var_3608_begin_0 = const()[name = string("op_3608_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_3608_end_0 = const()[name = string("op_3608_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_3608_end_mask_0 = const()[name = string("op_3608_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3608_squeeze_mask_0 = const()[name = string("op_3608_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3608_cast_fp16 = slice_by_index(begin = var_3608_begin_0, end = var_3608_end_0, end_mask = var_3608_end_mask_0, squeeze_mask = var_3608_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_3608_cast_fp16")];
+            bool var_3609_transpose_x_0 = const()[name = string("op_3609_transpose_x_0"), val = bool(false)];
+            bool var_3609_transpose_y_0 = const()[name = string("op_3609_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3609_cast_fp16 = matmul(transpose_x = var_3609_transpose_x_0, transpose_y = var_3609_transpose_y_0, x = var_3608_cast_fp16, y = state_131_cast_fp16)[name = string("op_3609_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_131_cast_fp16 = sub(x = var_3606_cast_fp16, y = var_3609_cast_fp16)[name = string("v_new_131_cast_fp16")];
+            tensor<int32, [4]> var_3612_begin_0 = const()[name = string("op_3612_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_3612_end_0 = const()[name = string("op_3612_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_3612_end_mask_0 = const()[name = string("op_3612_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3612_squeeze_mask_0 = const()[name = string("op_3612_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3612_cast_fp16 = slice_by_index(begin = var_3612_begin_0, end = var_3612_end_0, end_mask = var_3612_end_mask_0, squeeze_mask = var_3612_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_3612_cast_fp16")];
+            bool var_3613_transpose_x_0 = const()[name = string("op_3613_transpose_x_0"), val = bool(false)];
+            bool var_3613_transpose_y_0 = const()[name = string("op_3613_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3613_cast_fp16 = matmul(transpose_x = var_3613_transpose_x_0, transpose_y = var_3613_transpose_y_0, x = var_3612_cast_fp16, y = state_131_cast_fp16)[name = string("op_3613_cast_fp16")];
+            tensor<int32, [4]> var_3615_begin_0 = const()[name = string("op_3615_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_3615_end_0 = const()[name = string("op_3615_end_0"), val = tensor<int32, [4]>([32, 2, 64, 64])];
+            tensor<bool, [4]> var_3615_end_mask_0 = const()[name = string("op_3615_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3615_squeeze_mask_0 = const()[name = string("op_3615_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3615_cast_fp16 = slice_by_index(begin = var_3615_begin_0, end = var_3615_end_0, end_mask = var_3615_end_mask_0, squeeze_mask = var_3615_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_3615_cast_fp16")];
+            bool var_3616_transpose_x_0 = const()[name = string("op_3616_transpose_x_0"), val = bool(false)];
+            bool var_3616_transpose_y_0 = const()[name = string("op_3616_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3616_cast_fp16 = matmul(transpose_x = var_3616_transpose_x_0, transpose_y = var_3616_transpose_y_0, x = var_3615_cast_fp16, y = v_new_131_cast_fp16)[name = string("op_3616_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3617_cast_fp16 = add(x = var_3613_cast_fp16, y = var_3616_cast_fp16)[name = string("op_3617_cast_fp16")];
+            tensor<int32, [4]> var_3619_begin_0 = const()[name = string("op_3619_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_3619_end_0 = const()[name = string("op_3619_end_0"), val = tensor<int32, [4]>([32, 2, 1, 1])];
+            tensor<bool, [4]> var_3619_end_mask_0 = const()[name = string("op_3619_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3619_squeeze_mask_0 = const()[name = string("op_3619_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3619_cast_fp16 = slice_by_index(begin = var_3619_begin_0, end = var_3619_end_0, end_mask = var_3619_end_mask_0, squeeze_mask = var_3619_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_3619_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3620_cast_fp16 = mul(x = state_131_cast_fp16, y = var_3619_cast_fp16)[name = string("op_3620_cast_fp16")];
+            tensor<int32, [4]> var_3622_begin_0 = const()[name = string("op_3622_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_3622_end_0 = const()[name = string("op_3622_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_3622_end_mask_0 = const()[name = string("op_3622_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3622_squeeze_mask_0 = const()[name = string("op_3622_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3622_cast_fp16 = slice_by_index(begin = var_3622_begin_0, end = var_3622_end_0, end_mask = var_3622_end_mask_0, squeeze_mask = var_3622_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_3622_cast_fp16")];
+            bool var_3624_transpose_x_1 = const()[name = string("op_3624_transpose_x_1"), val = bool(true)];
+            bool var_3624_transpose_y_1 = const()[name = string("op_3624_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3624_cast_fp16 = matmul(transpose_x = var_3624_transpose_x_1, transpose_y = var_3624_transpose_y_1, x = var_3622_cast_fp16, y = v_new_131_cast_fp16)[name = string("op_3624_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_133_cast_fp16 = add(x = var_3620_cast_fp16, y = var_3624_cast_fp16)[name = string("state_133_cast_fp16")];
+            tensor<int32, [4]> var_3627_begin_0 = const()[name = string("op_3627_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_3627_end_0 = const()[name = string("op_3627_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_3627_end_mask_0 = const()[name = string("op_3627_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3627_squeeze_mask_0 = const()[name = string("op_3627_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3627_cast_fp16 = slice_by_index(begin = var_3627_begin_0, end = var_3627_end_0, end_mask = var_3627_end_mask_0, squeeze_mask = var_3627_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_3627_cast_fp16")];
+            tensor<int32, [4]> var_3629_begin_0 = const()[name = string("op_3629_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_3629_end_0 = const()[name = string("op_3629_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_3629_end_mask_0 = const()[name = string("op_3629_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3629_squeeze_mask_0 = const()[name = string("op_3629_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3629_cast_fp16 = slice_by_index(begin = var_3629_begin_0, end = var_3629_end_0, end_mask = var_3629_end_mask_0, squeeze_mask = var_3629_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_3629_cast_fp16")];
+            bool var_3630_transpose_x_0 = const()[name = string("op_3630_transpose_x_0"), val = bool(false)];
+            bool var_3630_transpose_y_0 = const()[name = string("op_3630_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3630_cast_fp16 = matmul(transpose_x = var_3630_transpose_x_0, transpose_y = var_3630_transpose_y_0, x = var_3629_cast_fp16, y = state_133_cast_fp16)[name = string("op_3630_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_133_cast_fp16 = sub(x = var_3627_cast_fp16, y = var_3630_cast_fp16)[name = string("v_new_133_cast_fp16")];
+            tensor<int32, [4]> var_3633_begin_0 = const()[name = string("op_3633_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_3633_end_0 = const()[name = string("op_3633_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_3633_end_mask_0 = const()[name = string("op_3633_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3633_squeeze_mask_0 = const()[name = string("op_3633_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3633_cast_fp16 = slice_by_index(begin = var_3633_begin_0, end = var_3633_end_0, end_mask = var_3633_end_mask_0, squeeze_mask = var_3633_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_3633_cast_fp16")];
+            bool var_3634_transpose_x_0 = const()[name = string("op_3634_transpose_x_0"), val = bool(false)];
+            bool var_3634_transpose_y_0 = const()[name = string("op_3634_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3634_cast_fp16 = matmul(transpose_x = var_3634_transpose_x_0, transpose_y = var_3634_transpose_y_0, x = var_3633_cast_fp16, y = state_133_cast_fp16)[name = string("op_3634_cast_fp16")];
+            tensor<int32, [4]> var_3636_begin_0 = const()[name = string("op_3636_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_3636_end_0 = const()[name = string("op_3636_end_0"), val = tensor<int32, [4]>([32, 3, 64, 64])];
+            tensor<bool, [4]> var_3636_end_mask_0 = const()[name = string("op_3636_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3636_squeeze_mask_0 = const()[name = string("op_3636_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3636_cast_fp16 = slice_by_index(begin = var_3636_begin_0, end = var_3636_end_0, end_mask = var_3636_end_mask_0, squeeze_mask = var_3636_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_3636_cast_fp16")];
+            bool var_3637_transpose_x_0 = const()[name = string("op_3637_transpose_x_0"), val = bool(false)];
+            bool var_3637_transpose_y_0 = const()[name = string("op_3637_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3637_cast_fp16 = matmul(transpose_x = var_3637_transpose_x_0, transpose_y = var_3637_transpose_y_0, x = var_3636_cast_fp16, y = v_new_133_cast_fp16)[name = string("op_3637_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3638_cast_fp16 = add(x = var_3634_cast_fp16, y = var_3637_cast_fp16)[name = string("op_3638_cast_fp16")];
+            tensor<int32, [4]> var_3640_begin_0 = const()[name = string("op_3640_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_3640_end_0 = const()[name = string("op_3640_end_0"), val = tensor<int32, [4]>([32, 3, 1, 1])];
+            tensor<bool, [4]> var_3640_end_mask_0 = const()[name = string("op_3640_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3640_squeeze_mask_0 = const()[name = string("op_3640_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3640_cast_fp16 = slice_by_index(begin = var_3640_begin_0, end = var_3640_end_0, end_mask = var_3640_end_mask_0, squeeze_mask = var_3640_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_3640_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3641_cast_fp16 = mul(x = state_133_cast_fp16, y = var_3640_cast_fp16)[name = string("op_3641_cast_fp16")];
+            tensor<int32, [4]> var_3643_begin_0 = const()[name = string("op_3643_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_3643_end_0 = const()[name = string("op_3643_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_3643_end_mask_0 = const()[name = string("op_3643_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3643_squeeze_mask_0 = const()[name = string("op_3643_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3643_cast_fp16 = slice_by_index(begin = var_3643_begin_0, end = var_3643_end_0, end_mask = var_3643_end_mask_0, squeeze_mask = var_3643_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_3643_cast_fp16")];
+            bool var_3645_transpose_x_1 = const()[name = string("op_3645_transpose_x_1"), val = bool(true)];
+            bool var_3645_transpose_y_1 = const()[name = string("op_3645_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3645_cast_fp16 = matmul(transpose_x = var_3645_transpose_x_1, transpose_y = var_3645_transpose_y_1, x = var_3643_cast_fp16, y = v_new_133_cast_fp16)[name = string("op_3645_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_135_cast_fp16 = add(x = var_3641_cast_fp16, y = var_3645_cast_fp16)[name = string("state_135_cast_fp16")];
+            tensor<int32, [4]> var_3648_begin_0 = const()[name = string("op_3648_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_3648_end_0 = const()[name = string("op_3648_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_3648_end_mask_0 = const()[name = string("op_3648_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3648_squeeze_mask_0 = const()[name = string("op_3648_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3648_cast_fp16 = slice_by_index(begin = var_3648_begin_0, end = var_3648_end_0, end_mask = var_3648_end_mask_0, squeeze_mask = var_3648_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_3648_cast_fp16")];
+            tensor<int32, [4]> var_3650_begin_0 = const()[name = string("op_3650_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_3650_end_0 = const()[name = string("op_3650_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_3650_end_mask_0 = const()[name = string("op_3650_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3650_squeeze_mask_0 = const()[name = string("op_3650_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3650_cast_fp16 = slice_by_index(begin = var_3650_begin_0, end = var_3650_end_0, end_mask = var_3650_end_mask_0, squeeze_mask = var_3650_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_3650_cast_fp16")];
+            bool var_3651_transpose_x_0 = const()[name = string("op_3651_transpose_x_0"), val = bool(false)];
+            bool var_3651_transpose_y_0 = const()[name = string("op_3651_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3651_cast_fp16 = matmul(transpose_x = var_3651_transpose_x_0, transpose_y = var_3651_transpose_y_0, x = var_3650_cast_fp16, y = state_135_cast_fp16)[name = string("op_3651_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_135_cast_fp16 = sub(x = var_3648_cast_fp16, y = var_3651_cast_fp16)[name = string("v_new_135_cast_fp16")];
+            tensor<int32, [4]> var_3654_begin_0 = const()[name = string("op_3654_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_3654_end_0 = const()[name = string("op_3654_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_3654_end_mask_0 = const()[name = string("op_3654_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3654_squeeze_mask_0 = const()[name = string("op_3654_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3654_cast_fp16 = slice_by_index(begin = var_3654_begin_0, end = var_3654_end_0, end_mask = var_3654_end_mask_0, squeeze_mask = var_3654_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_3654_cast_fp16")];
+            bool var_3655_transpose_x_0 = const()[name = string("op_3655_transpose_x_0"), val = bool(false)];
+            bool var_3655_transpose_y_0 = const()[name = string("op_3655_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3655_cast_fp16 = matmul(transpose_x = var_3655_transpose_x_0, transpose_y = var_3655_transpose_y_0, x = var_3654_cast_fp16, y = state_135_cast_fp16)[name = string("op_3655_cast_fp16")];
+            tensor<int32, [4]> var_3657_begin_0 = const()[name = string("op_3657_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_3657_end_0 = const()[name = string("op_3657_end_0"), val = tensor<int32, [4]>([32, 4, 64, 64])];
+            tensor<bool, [4]> var_3657_end_mask_0 = const()[name = string("op_3657_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3657_squeeze_mask_0 = const()[name = string("op_3657_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3657_cast_fp16 = slice_by_index(begin = var_3657_begin_0, end = var_3657_end_0, end_mask = var_3657_end_mask_0, squeeze_mask = var_3657_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_3657_cast_fp16")];
+            bool var_3658_transpose_x_0 = const()[name = string("op_3658_transpose_x_0"), val = bool(false)];
+            bool var_3658_transpose_y_0 = const()[name = string("op_3658_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3658_cast_fp16 = matmul(transpose_x = var_3658_transpose_x_0, transpose_y = var_3658_transpose_y_0, x = var_3657_cast_fp16, y = v_new_135_cast_fp16)[name = string("op_3658_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3659_cast_fp16 = add(x = var_3655_cast_fp16, y = var_3658_cast_fp16)[name = string("op_3659_cast_fp16")];
+            tensor<int32, [4]> var_3661_begin_0 = const()[name = string("op_3661_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_3661_end_0 = const()[name = string("op_3661_end_0"), val = tensor<int32, [4]>([32, 4, 1, 1])];
+            tensor<bool, [4]> var_3661_end_mask_0 = const()[name = string("op_3661_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3661_squeeze_mask_0 = const()[name = string("op_3661_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3661_cast_fp16 = slice_by_index(begin = var_3661_begin_0, end = var_3661_end_0, end_mask = var_3661_end_mask_0, squeeze_mask = var_3661_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_3661_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3662_cast_fp16 = mul(x = state_135_cast_fp16, y = var_3661_cast_fp16)[name = string("op_3662_cast_fp16")];
+            tensor<int32, [4]> var_3664_begin_0 = const()[name = string("op_3664_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_3664_end_0 = const()[name = string("op_3664_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_3664_end_mask_0 = const()[name = string("op_3664_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3664_squeeze_mask_0 = const()[name = string("op_3664_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3664_cast_fp16 = slice_by_index(begin = var_3664_begin_0, end = var_3664_end_0, end_mask = var_3664_end_mask_0, squeeze_mask = var_3664_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_3664_cast_fp16")];
+            bool var_3666_transpose_x_1 = const()[name = string("op_3666_transpose_x_1"), val = bool(true)];
+            bool var_3666_transpose_y_1 = const()[name = string("op_3666_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3666_cast_fp16 = matmul(transpose_x = var_3666_transpose_x_1, transpose_y = var_3666_transpose_y_1, x = var_3664_cast_fp16, y = v_new_135_cast_fp16)[name = string("op_3666_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_137_cast_fp16 = add(x = var_3662_cast_fp16, y = var_3666_cast_fp16)[name = string("state_137_cast_fp16")];
+            tensor<int32, [4]> var_3669_begin_0 = const()[name = string("op_3669_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_3669_end_0 = const()[name = string("op_3669_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_3669_end_mask_0 = const()[name = string("op_3669_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3669_squeeze_mask_0 = const()[name = string("op_3669_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3669_cast_fp16 = slice_by_index(begin = var_3669_begin_0, end = var_3669_end_0, end_mask = var_3669_end_mask_0, squeeze_mask = var_3669_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_3669_cast_fp16")];
+            tensor<int32, [4]> var_3671_begin_0 = const()[name = string("op_3671_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_3671_end_0 = const()[name = string("op_3671_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_3671_end_mask_0 = const()[name = string("op_3671_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3671_squeeze_mask_0 = const()[name = string("op_3671_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3671_cast_fp16 = slice_by_index(begin = var_3671_begin_0, end = var_3671_end_0, end_mask = var_3671_end_mask_0, squeeze_mask = var_3671_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_3671_cast_fp16")];
+            bool var_3672_transpose_x_0 = const()[name = string("op_3672_transpose_x_0"), val = bool(false)];
+            bool var_3672_transpose_y_0 = const()[name = string("op_3672_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3672_cast_fp16 = matmul(transpose_x = var_3672_transpose_x_0, transpose_y = var_3672_transpose_y_0, x = var_3671_cast_fp16, y = state_137_cast_fp16)[name = string("op_3672_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_137_cast_fp16 = sub(x = var_3669_cast_fp16, y = var_3672_cast_fp16)[name = string("v_new_137_cast_fp16")];
+            tensor<int32, [4]> var_3675_begin_0 = const()[name = string("op_3675_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_3675_end_0 = const()[name = string("op_3675_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_3675_end_mask_0 = const()[name = string("op_3675_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3675_squeeze_mask_0 = const()[name = string("op_3675_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3675_cast_fp16 = slice_by_index(begin = var_3675_begin_0, end = var_3675_end_0, end_mask = var_3675_end_mask_0, squeeze_mask = var_3675_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_3675_cast_fp16")];
+            bool var_3676_transpose_x_0 = const()[name = string("op_3676_transpose_x_0"), val = bool(false)];
+            bool var_3676_transpose_y_0 = const()[name = string("op_3676_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3676_cast_fp16 = matmul(transpose_x = var_3676_transpose_x_0, transpose_y = var_3676_transpose_y_0, x = var_3675_cast_fp16, y = state_137_cast_fp16)[name = string("op_3676_cast_fp16")];
+            tensor<int32, [4]> var_3678_begin_0 = const()[name = string("op_3678_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_3678_end_0 = const()[name = string("op_3678_end_0"), val = tensor<int32, [4]>([32, 5, 64, 64])];
+            tensor<bool, [4]> var_3678_end_mask_0 = const()[name = string("op_3678_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3678_squeeze_mask_0 = const()[name = string("op_3678_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3678_cast_fp16 = slice_by_index(begin = var_3678_begin_0, end = var_3678_end_0, end_mask = var_3678_end_mask_0, squeeze_mask = var_3678_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_3678_cast_fp16")];
+            bool var_3679_transpose_x_0 = const()[name = string("op_3679_transpose_x_0"), val = bool(false)];
+            bool var_3679_transpose_y_0 = const()[name = string("op_3679_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3679_cast_fp16 = matmul(transpose_x = var_3679_transpose_x_0, transpose_y = var_3679_transpose_y_0, x = var_3678_cast_fp16, y = v_new_137_cast_fp16)[name = string("op_3679_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3680_cast_fp16 = add(x = var_3676_cast_fp16, y = var_3679_cast_fp16)[name = string("op_3680_cast_fp16")];
+            tensor<int32, [4]> var_3682_begin_0 = const()[name = string("op_3682_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_3682_end_0 = const()[name = string("op_3682_end_0"), val = tensor<int32, [4]>([32, 5, 1, 1])];
+            tensor<bool, [4]> var_3682_end_mask_0 = const()[name = string("op_3682_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3682_squeeze_mask_0 = const()[name = string("op_3682_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3682_cast_fp16 = slice_by_index(begin = var_3682_begin_0, end = var_3682_end_0, end_mask = var_3682_end_mask_0, squeeze_mask = var_3682_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_3682_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3683_cast_fp16 = mul(x = state_137_cast_fp16, y = var_3682_cast_fp16)[name = string("op_3683_cast_fp16")];
+            tensor<int32, [4]> var_3685_begin_0 = const()[name = string("op_3685_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_3685_end_0 = const()[name = string("op_3685_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_3685_end_mask_0 = const()[name = string("op_3685_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3685_squeeze_mask_0 = const()[name = string("op_3685_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3685_cast_fp16 = slice_by_index(begin = var_3685_begin_0, end = var_3685_end_0, end_mask = var_3685_end_mask_0, squeeze_mask = var_3685_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_3685_cast_fp16")];
+            bool var_3687_transpose_x_1 = const()[name = string("op_3687_transpose_x_1"), val = bool(true)];
+            bool var_3687_transpose_y_1 = const()[name = string("op_3687_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3687_cast_fp16 = matmul(transpose_x = var_3687_transpose_x_1, transpose_y = var_3687_transpose_y_1, x = var_3685_cast_fp16, y = v_new_137_cast_fp16)[name = string("op_3687_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_139_cast_fp16 = add(x = var_3683_cast_fp16, y = var_3687_cast_fp16)[name = string("state_139_cast_fp16")];
+            tensor<int32, [4]> var_3690_begin_0 = const()[name = string("op_3690_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_3690_end_0 = const()[name = string("op_3690_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_3690_end_mask_0 = const()[name = string("op_3690_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3690_squeeze_mask_0 = const()[name = string("op_3690_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3690_cast_fp16 = slice_by_index(begin = var_3690_begin_0, end = var_3690_end_0, end_mask = var_3690_end_mask_0, squeeze_mask = var_3690_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_3690_cast_fp16")];
+            tensor<int32, [4]> var_3692_begin_0 = const()[name = string("op_3692_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_3692_end_0 = const()[name = string("op_3692_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_3692_end_mask_0 = const()[name = string("op_3692_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3692_squeeze_mask_0 = const()[name = string("op_3692_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3692_cast_fp16 = slice_by_index(begin = var_3692_begin_0, end = var_3692_end_0, end_mask = var_3692_end_mask_0, squeeze_mask = var_3692_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_3692_cast_fp16")];
+            bool var_3693_transpose_x_0 = const()[name = string("op_3693_transpose_x_0"), val = bool(false)];
+            bool var_3693_transpose_y_0 = const()[name = string("op_3693_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3693_cast_fp16 = matmul(transpose_x = var_3693_transpose_x_0, transpose_y = var_3693_transpose_y_0, x = var_3692_cast_fp16, y = state_139_cast_fp16)[name = string("op_3693_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_139_cast_fp16 = sub(x = var_3690_cast_fp16, y = var_3693_cast_fp16)[name = string("v_new_139_cast_fp16")];
+            tensor<int32, [4]> var_3696_begin_0 = const()[name = string("op_3696_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_3696_end_0 = const()[name = string("op_3696_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_3696_end_mask_0 = const()[name = string("op_3696_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3696_squeeze_mask_0 = const()[name = string("op_3696_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3696_cast_fp16 = slice_by_index(begin = var_3696_begin_0, end = var_3696_end_0, end_mask = var_3696_end_mask_0, squeeze_mask = var_3696_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_3696_cast_fp16")];
+            bool var_3697_transpose_x_0 = const()[name = string("op_3697_transpose_x_0"), val = bool(false)];
+            bool var_3697_transpose_y_0 = const()[name = string("op_3697_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3697_cast_fp16 = matmul(transpose_x = var_3697_transpose_x_0, transpose_y = var_3697_transpose_y_0, x = var_3696_cast_fp16, y = state_139_cast_fp16)[name = string("op_3697_cast_fp16")];
+            tensor<int32, [4]> var_3699_begin_0 = const()[name = string("op_3699_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_3699_end_0 = const()[name = string("op_3699_end_0"), val = tensor<int32, [4]>([32, 6, 64, 64])];
+            tensor<bool, [4]> var_3699_end_mask_0 = const()[name = string("op_3699_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3699_squeeze_mask_0 = const()[name = string("op_3699_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3699_cast_fp16 = slice_by_index(begin = var_3699_begin_0, end = var_3699_end_0, end_mask = var_3699_end_mask_0, squeeze_mask = var_3699_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_3699_cast_fp16")];
+            bool var_3700_transpose_x_0 = const()[name = string("op_3700_transpose_x_0"), val = bool(false)];
+            bool var_3700_transpose_y_0 = const()[name = string("op_3700_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3700_cast_fp16 = matmul(transpose_x = var_3700_transpose_x_0, transpose_y = var_3700_transpose_y_0, x = var_3699_cast_fp16, y = v_new_139_cast_fp16)[name = string("op_3700_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3701_cast_fp16 = add(x = var_3697_cast_fp16, y = var_3700_cast_fp16)[name = string("op_3701_cast_fp16")];
+            tensor<int32, [4]> var_3703_begin_0 = const()[name = string("op_3703_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_3703_end_0 = const()[name = string("op_3703_end_0"), val = tensor<int32, [4]>([32, 6, 1, 1])];
+            tensor<bool, [4]> var_3703_end_mask_0 = const()[name = string("op_3703_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3703_squeeze_mask_0 = const()[name = string("op_3703_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3703_cast_fp16 = slice_by_index(begin = var_3703_begin_0, end = var_3703_end_0, end_mask = var_3703_end_mask_0, squeeze_mask = var_3703_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_3703_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3704_cast_fp16 = mul(x = state_139_cast_fp16, y = var_3703_cast_fp16)[name = string("op_3704_cast_fp16")];
+            tensor<int32, [4]> var_3706_begin_0 = const()[name = string("op_3706_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_3706_end_0 = const()[name = string("op_3706_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_3706_end_mask_0 = const()[name = string("op_3706_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3706_squeeze_mask_0 = const()[name = string("op_3706_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3706_cast_fp16 = slice_by_index(begin = var_3706_begin_0, end = var_3706_end_0, end_mask = var_3706_end_mask_0, squeeze_mask = var_3706_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_3706_cast_fp16")];
+            bool var_3708_transpose_x_1 = const()[name = string("op_3708_transpose_x_1"), val = bool(true)];
+            bool var_3708_transpose_y_1 = const()[name = string("op_3708_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3708_cast_fp16 = matmul(transpose_x = var_3708_transpose_x_1, transpose_y = var_3708_transpose_y_1, x = var_3706_cast_fp16, y = v_new_139_cast_fp16)[name = string("op_3708_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_141_cast_fp16 = add(x = var_3704_cast_fp16, y = var_3708_cast_fp16)[name = string("state_141_cast_fp16")];
+            tensor<int32, [4]> var_3711_begin_0 = const()[name = string("op_3711_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_3711_end_0 = const()[name = string("op_3711_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_3711_end_mask_0 = const()[name = string("op_3711_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3711_squeeze_mask_0 = const()[name = string("op_3711_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3711_cast_fp16 = slice_by_index(begin = var_3711_begin_0, end = var_3711_end_0, end_mask = var_3711_end_mask_0, squeeze_mask = var_3711_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_3711_cast_fp16")];
+            tensor<int32, [4]> var_3713_begin_0 = const()[name = string("op_3713_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_3713_end_0 = const()[name = string("op_3713_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_3713_end_mask_0 = const()[name = string("op_3713_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3713_squeeze_mask_0 = const()[name = string("op_3713_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3713_cast_fp16 = slice_by_index(begin = var_3713_begin_0, end = var_3713_end_0, end_mask = var_3713_end_mask_0, squeeze_mask = var_3713_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_3713_cast_fp16")];
+            bool var_3714_transpose_x_0 = const()[name = string("op_3714_transpose_x_0"), val = bool(false)];
+            bool var_3714_transpose_y_0 = const()[name = string("op_3714_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3714_cast_fp16 = matmul(transpose_x = var_3714_transpose_x_0, transpose_y = var_3714_transpose_y_0, x = var_3713_cast_fp16, y = state_141_cast_fp16)[name = string("op_3714_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_141_cast_fp16 = sub(x = var_3711_cast_fp16, y = var_3714_cast_fp16)[name = string("v_new_141_cast_fp16")];
+            tensor<int32, [4]> var_3717_begin_0 = const()[name = string("op_3717_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_3717_end_0 = const()[name = string("op_3717_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_3717_end_mask_0 = const()[name = string("op_3717_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3717_squeeze_mask_0 = const()[name = string("op_3717_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3717_cast_fp16 = slice_by_index(begin = var_3717_begin_0, end = var_3717_end_0, end_mask = var_3717_end_mask_0, squeeze_mask = var_3717_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_3717_cast_fp16")];
+            bool var_3718_transpose_x_0 = const()[name = string("op_3718_transpose_x_0"), val = bool(false)];
+            bool var_3718_transpose_y_0 = const()[name = string("op_3718_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3718_cast_fp16 = matmul(transpose_x = var_3718_transpose_x_0, transpose_y = var_3718_transpose_y_0, x = var_3717_cast_fp16, y = state_141_cast_fp16)[name = string("op_3718_cast_fp16")];
+            tensor<int32, [4]> var_3720_begin_0 = const()[name = string("op_3720_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_3720_end_0 = const()[name = string("op_3720_end_0"), val = tensor<int32, [4]>([32, 7, 64, 64])];
+            tensor<bool, [4]> var_3720_end_mask_0 = const()[name = string("op_3720_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3720_squeeze_mask_0 = const()[name = string("op_3720_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3720_cast_fp16 = slice_by_index(begin = var_3720_begin_0, end = var_3720_end_0, end_mask = var_3720_end_mask_0, squeeze_mask = var_3720_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_3720_cast_fp16")];
+            bool var_3721_transpose_x_0 = const()[name = string("op_3721_transpose_x_0"), val = bool(false)];
+            bool var_3721_transpose_y_0 = const()[name = string("op_3721_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3721_cast_fp16 = matmul(transpose_x = var_3721_transpose_x_0, transpose_y = var_3721_transpose_y_0, x = var_3720_cast_fp16, y = v_new_141_cast_fp16)[name = string("op_3721_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3722_cast_fp16 = add(x = var_3718_cast_fp16, y = var_3721_cast_fp16)[name = string("op_3722_cast_fp16")];
+            tensor<int32, [4]> var_3724_begin_0 = const()[name = string("op_3724_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_3724_end_0 = const()[name = string("op_3724_end_0"), val = tensor<int32, [4]>([32, 7, 1, 1])];
+            tensor<bool, [4]> var_3724_end_mask_0 = const()[name = string("op_3724_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3724_squeeze_mask_0 = const()[name = string("op_3724_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3724_cast_fp16 = slice_by_index(begin = var_3724_begin_0, end = var_3724_end_0, end_mask = var_3724_end_mask_0, squeeze_mask = var_3724_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_3724_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3725_cast_fp16 = mul(x = state_141_cast_fp16, y = var_3724_cast_fp16)[name = string("op_3725_cast_fp16")];
+            tensor<int32, [4]> var_3727_begin_0 = const()[name = string("op_3727_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_3727_end_0 = const()[name = string("op_3727_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_3727_end_mask_0 = const()[name = string("op_3727_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3727_squeeze_mask_0 = const()[name = string("op_3727_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3727_cast_fp16 = slice_by_index(begin = var_3727_begin_0, end = var_3727_end_0, end_mask = var_3727_end_mask_0, squeeze_mask = var_3727_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_3727_cast_fp16")];
+            bool var_3729_transpose_x_1 = const()[name = string("op_3729_transpose_x_1"), val = bool(true)];
+            bool var_3729_transpose_y_1 = const()[name = string("op_3729_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3729_cast_fp16 = matmul(transpose_x = var_3729_transpose_x_1, transpose_y = var_3729_transpose_y_1, x = var_3727_cast_fp16, y = v_new_141_cast_fp16)[name = string("op_3729_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_143_cast_fp16 = add(x = var_3725_cast_fp16, y = var_3729_cast_fp16)[name = string("state_143_cast_fp16")];
+            tensor<int32, [4]> var_3732_begin_0 = const()[name = string("op_3732_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_3732_end_0 = const()[name = string("op_3732_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_3732_end_mask_0 = const()[name = string("op_3732_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3732_squeeze_mask_0 = const()[name = string("op_3732_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3732_cast_fp16 = slice_by_index(begin = var_3732_begin_0, end = var_3732_end_0, end_mask = var_3732_end_mask_0, squeeze_mask = var_3732_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_3732_cast_fp16")];
+            tensor<int32, [4]> var_3734_begin_0 = const()[name = string("op_3734_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_3734_end_0 = const()[name = string("op_3734_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_3734_end_mask_0 = const()[name = string("op_3734_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3734_squeeze_mask_0 = const()[name = string("op_3734_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3734_cast_fp16 = slice_by_index(begin = var_3734_begin_0, end = var_3734_end_0, end_mask = var_3734_end_mask_0, squeeze_mask = var_3734_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_3734_cast_fp16")];
+            bool var_3735_transpose_x_0 = const()[name = string("op_3735_transpose_x_0"), val = bool(false)];
+            bool var_3735_transpose_y_0 = const()[name = string("op_3735_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3735_cast_fp16 = matmul(transpose_x = var_3735_transpose_x_0, transpose_y = var_3735_transpose_y_0, x = var_3734_cast_fp16, y = state_143_cast_fp16)[name = string("op_3735_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_143_cast_fp16 = sub(x = var_3732_cast_fp16, y = var_3735_cast_fp16)[name = string("v_new_143_cast_fp16")];
+            tensor<int32, [4]> var_3738_begin_0 = const()[name = string("op_3738_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_3738_end_0 = const()[name = string("op_3738_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_3738_end_mask_0 = const()[name = string("op_3738_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3738_squeeze_mask_0 = const()[name = string("op_3738_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3738_cast_fp16 = slice_by_index(begin = var_3738_begin_0, end = var_3738_end_0, end_mask = var_3738_end_mask_0, squeeze_mask = var_3738_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_3738_cast_fp16")];
+            bool var_3739_transpose_x_0 = const()[name = string("op_3739_transpose_x_0"), val = bool(false)];
+            bool var_3739_transpose_y_0 = const()[name = string("op_3739_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3739_cast_fp16 = matmul(transpose_x = var_3739_transpose_x_0, transpose_y = var_3739_transpose_y_0, x = var_3738_cast_fp16, y = state_143_cast_fp16)[name = string("op_3739_cast_fp16")];
+            tensor<int32, [4]> var_3741_begin_0 = const()[name = string("op_3741_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_3741_end_0 = const()[name = string("op_3741_end_0"), val = tensor<int32, [4]>([32, 8, 64, 64])];
+            tensor<bool, [4]> var_3741_end_mask_0 = const()[name = string("op_3741_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3741_squeeze_mask_0 = const()[name = string("op_3741_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3741_cast_fp16 = slice_by_index(begin = var_3741_begin_0, end = var_3741_end_0, end_mask = var_3741_end_mask_0, squeeze_mask = var_3741_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_3741_cast_fp16")];
+            bool var_3742_transpose_x_0 = const()[name = string("op_3742_transpose_x_0"), val = bool(false)];
+            bool var_3742_transpose_y_0 = const()[name = string("op_3742_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3742_cast_fp16 = matmul(transpose_x = var_3742_transpose_x_0, transpose_y = var_3742_transpose_y_0, x = var_3741_cast_fp16, y = v_new_143_cast_fp16)[name = string("op_3742_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3743_cast_fp16 = add(x = var_3739_cast_fp16, y = var_3742_cast_fp16)[name = string("op_3743_cast_fp16")];
+            tensor<int32, [4]> var_3745_begin_0 = const()[name = string("op_3745_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_3745_end_0 = const()[name = string("op_3745_end_0"), val = tensor<int32, [4]>([32, 8, 1, 1])];
+            tensor<bool, [4]> var_3745_end_mask_0 = const()[name = string("op_3745_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3745_squeeze_mask_0 = const()[name = string("op_3745_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3745_cast_fp16 = slice_by_index(begin = var_3745_begin_0, end = var_3745_end_0, end_mask = var_3745_end_mask_0, squeeze_mask = var_3745_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_3745_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3746_cast_fp16 = mul(x = state_143_cast_fp16, y = var_3745_cast_fp16)[name = string("op_3746_cast_fp16")];
+            tensor<int32, [4]> var_3748_begin_0 = const()[name = string("op_3748_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_3748_end_0 = const()[name = string("op_3748_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_3748_end_mask_0 = const()[name = string("op_3748_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3748_squeeze_mask_0 = const()[name = string("op_3748_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3748_cast_fp16 = slice_by_index(begin = var_3748_begin_0, end = var_3748_end_0, end_mask = var_3748_end_mask_0, squeeze_mask = var_3748_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_3748_cast_fp16")];
+            bool var_3750_transpose_x_1 = const()[name = string("op_3750_transpose_x_1"), val = bool(true)];
+            bool var_3750_transpose_y_1 = const()[name = string("op_3750_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3750_cast_fp16 = matmul(transpose_x = var_3750_transpose_x_1, transpose_y = var_3750_transpose_y_1, x = var_3748_cast_fp16, y = v_new_143_cast_fp16)[name = string("op_3750_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_145_cast_fp16 = add(x = var_3746_cast_fp16, y = var_3750_cast_fp16)[name = string("state_145_cast_fp16")];
+            tensor<int32, [4]> var_3753_begin_0 = const()[name = string("op_3753_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_3753_end_0 = const()[name = string("op_3753_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_3753_end_mask_0 = const()[name = string("op_3753_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3753_squeeze_mask_0 = const()[name = string("op_3753_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3753_cast_fp16 = slice_by_index(begin = var_3753_begin_0, end = var_3753_end_0, end_mask = var_3753_end_mask_0, squeeze_mask = var_3753_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_3753_cast_fp16")];
+            tensor<int32, [4]> var_3755_begin_0 = const()[name = string("op_3755_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_3755_end_0 = const()[name = string("op_3755_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_3755_end_mask_0 = const()[name = string("op_3755_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3755_squeeze_mask_0 = const()[name = string("op_3755_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3755_cast_fp16 = slice_by_index(begin = var_3755_begin_0, end = var_3755_end_0, end_mask = var_3755_end_mask_0, squeeze_mask = var_3755_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_3755_cast_fp16")];
+            bool var_3756_transpose_x_0 = const()[name = string("op_3756_transpose_x_0"), val = bool(false)];
+            bool var_3756_transpose_y_0 = const()[name = string("op_3756_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3756_cast_fp16 = matmul(transpose_x = var_3756_transpose_x_0, transpose_y = var_3756_transpose_y_0, x = var_3755_cast_fp16, y = state_145_cast_fp16)[name = string("op_3756_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_145_cast_fp16 = sub(x = var_3753_cast_fp16, y = var_3756_cast_fp16)[name = string("v_new_145_cast_fp16")];
+            tensor<int32, [4]> var_3759_begin_0 = const()[name = string("op_3759_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_3759_end_0 = const()[name = string("op_3759_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_3759_end_mask_0 = const()[name = string("op_3759_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3759_squeeze_mask_0 = const()[name = string("op_3759_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3759_cast_fp16 = slice_by_index(begin = var_3759_begin_0, end = var_3759_end_0, end_mask = var_3759_end_mask_0, squeeze_mask = var_3759_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_3759_cast_fp16")];
+            bool var_3760_transpose_x_0 = const()[name = string("op_3760_transpose_x_0"), val = bool(false)];
+            bool var_3760_transpose_y_0 = const()[name = string("op_3760_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3760_cast_fp16 = matmul(transpose_x = var_3760_transpose_x_0, transpose_y = var_3760_transpose_y_0, x = var_3759_cast_fp16, y = state_145_cast_fp16)[name = string("op_3760_cast_fp16")];
+            tensor<int32, [4]> var_3762_begin_0 = const()[name = string("op_3762_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_3762_end_0 = const()[name = string("op_3762_end_0"), val = tensor<int32, [4]>([32, 9, 64, 64])];
+            tensor<bool, [4]> var_3762_end_mask_0 = const()[name = string("op_3762_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3762_squeeze_mask_0 = const()[name = string("op_3762_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3762_cast_fp16 = slice_by_index(begin = var_3762_begin_0, end = var_3762_end_0, end_mask = var_3762_end_mask_0, squeeze_mask = var_3762_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_3762_cast_fp16")];
+            bool var_3763_transpose_x_0 = const()[name = string("op_3763_transpose_x_0"), val = bool(false)];
+            bool var_3763_transpose_y_0 = const()[name = string("op_3763_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3763_cast_fp16 = matmul(transpose_x = var_3763_transpose_x_0, transpose_y = var_3763_transpose_y_0, x = var_3762_cast_fp16, y = v_new_145_cast_fp16)[name = string("op_3763_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3764_cast_fp16 = add(x = var_3760_cast_fp16, y = var_3763_cast_fp16)[name = string("op_3764_cast_fp16")];
+            tensor<int32, [4]> var_3766_begin_0 = const()[name = string("op_3766_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_3766_end_0 = const()[name = string("op_3766_end_0"), val = tensor<int32, [4]>([32, 9, 1, 1])];
+            tensor<bool, [4]> var_3766_end_mask_0 = const()[name = string("op_3766_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3766_squeeze_mask_0 = const()[name = string("op_3766_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3766_cast_fp16 = slice_by_index(begin = var_3766_begin_0, end = var_3766_end_0, end_mask = var_3766_end_mask_0, squeeze_mask = var_3766_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_3766_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3767_cast_fp16 = mul(x = state_145_cast_fp16, y = var_3766_cast_fp16)[name = string("op_3767_cast_fp16")];
+            tensor<int32, [4]> var_3769_begin_0 = const()[name = string("op_3769_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_3769_end_0 = const()[name = string("op_3769_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_3769_end_mask_0 = const()[name = string("op_3769_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3769_squeeze_mask_0 = const()[name = string("op_3769_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3769_cast_fp16 = slice_by_index(begin = var_3769_begin_0, end = var_3769_end_0, end_mask = var_3769_end_mask_0, squeeze_mask = var_3769_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_3769_cast_fp16")];
+            bool var_3771_transpose_x_1 = const()[name = string("op_3771_transpose_x_1"), val = bool(true)];
+            bool var_3771_transpose_y_1 = const()[name = string("op_3771_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3771_cast_fp16 = matmul(transpose_x = var_3771_transpose_x_1, transpose_y = var_3771_transpose_y_1, x = var_3769_cast_fp16, y = v_new_145_cast_fp16)[name = string("op_3771_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_147_cast_fp16 = add(x = var_3767_cast_fp16, y = var_3771_cast_fp16)[name = string("state_147_cast_fp16")];
+            tensor<int32, [4]> var_3774_begin_0 = const()[name = string("op_3774_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_3774_end_0 = const()[name = string("op_3774_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_3774_end_mask_0 = const()[name = string("op_3774_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3774_squeeze_mask_0 = const()[name = string("op_3774_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3774_cast_fp16 = slice_by_index(begin = var_3774_begin_0, end = var_3774_end_0, end_mask = var_3774_end_mask_0, squeeze_mask = var_3774_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_3774_cast_fp16")];
+            tensor<int32, [4]> var_3776_begin_0 = const()[name = string("op_3776_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_3776_end_0 = const()[name = string("op_3776_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_3776_end_mask_0 = const()[name = string("op_3776_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3776_squeeze_mask_0 = const()[name = string("op_3776_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3776_cast_fp16 = slice_by_index(begin = var_3776_begin_0, end = var_3776_end_0, end_mask = var_3776_end_mask_0, squeeze_mask = var_3776_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_3776_cast_fp16")];
+            bool var_3777_transpose_x_0 = const()[name = string("op_3777_transpose_x_0"), val = bool(false)];
+            bool var_3777_transpose_y_0 = const()[name = string("op_3777_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3777_cast_fp16 = matmul(transpose_x = var_3777_transpose_x_0, transpose_y = var_3777_transpose_y_0, x = var_3776_cast_fp16, y = state_147_cast_fp16)[name = string("op_3777_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_147_cast_fp16 = sub(x = var_3774_cast_fp16, y = var_3777_cast_fp16)[name = string("v_new_147_cast_fp16")];
+            tensor<int32, [4]> var_3780_begin_0 = const()[name = string("op_3780_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_3780_end_0 = const()[name = string("op_3780_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_3780_end_mask_0 = const()[name = string("op_3780_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3780_squeeze_mask_0 = const()[name = string("op_3780_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3780_cast_fp16 = slice_by_index(begin = var_3780_begin_0, end = var_3780_end_0, end_mask = var_3780_end_mask_0, squeeze_mask = var_3780_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_3780_cast_fp16")];
+            bool var_3781_transpose_x_0 = const()[name = string("op_3781_transpose_x_0"), val = bool(false)];
+            bool var_3781_transpose_y_0 = const()[name = string("op_3781_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3781_cast_fp16 = matmul(transpose_x = var_3781_transpose_x_0, transpose_y = var_3781_transpose_y_0, x = var_3780_cast_fp16, y = state_147_cast_fp16)[name = string("op_3781_cast_fp16")];
+            tensor<int32, [4]> var_3783_begin_0 = const()[name = string("op_3783_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_3783_end_0 = const()[name = string("op_3783_end_0"), val = tensor<int32, [4]>([32, 10, 64, 64])];
+            tensor<bool, [4]> var_3783_end_mask_0 = const()[name = string("op_3783_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3783_squeeze_mask_0 = const()[name = string("op_3783_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3783_cast_fp16 = slice_by_index(begin = var_3783_begin_0, end = var_3783_end_0, end_mask = var_3783_end_mask_0, squeeze_mask = var_3783_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_3783_cast_fp16")];
+            bool var_3784_transpose_x_0 = const()[name = string("op_3784_transpose_x_0"), val = bool(false)];
+            bool var_3784_transpose_y_0 = const()[name = string("op_3784_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3784_cast_fp16 = matmul(transpose_x = var_3784_transpose_x_0, transpose_y = var_3784_transpose_y_0, x = var_3783_cast_fp16, y = v_new_147_cast_fp16)[name = string("op_3784_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3785_cast_fp16 = add(x = var_3781_cast_fp16, y = var_3784_cast_fp16)[name = string("op_3785_cast_fp16")];
+            tensor<int32, [4]> var_3787_begin_0 = const()[name = string("op_3787_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_3787_end_0 = const()[name = string("op_3787_end_0"), val = tensor<int32, [4]>([32, 10, 1, 1])];
+            tensor<bool, [4]> var_3787_end_mask_0 = const()[name = string("op_3787_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3787_squeeze_mask_0 = const()[name = string("op_3787_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3787_cast_fp16 = slice_by_index(begin = var_3787_begin_0, end = var_3787_end_0, end_mask = var_3787_end_mask_0, squeeze_mask = var_3787_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_3787_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3788_cast_fp16 = mul(x = state_147_cast_fp16, y = var_3787_cast_fp16)[name = string("op_3788_cast_fp16")];
+            tensor<int32, [4]> var_3790_begin_0 = const()[name = string("op_3790_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_3790_end_0 = const()[name = string("op_3790_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_3790_end_mask_0 = const()[name = string("op_3790_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3790_squeeze_mask_0 = const()[name = string("op_3790_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3790_cast_fp16 = slice_by_index(begin = var_3790_begin_0, end = var_3790_end_0, end_mask = var_3790_end_mask_0, squeeze_mask = var_3790_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_3790_cast_fp16")];
+            bool var_3792_transpose_x_1 = const()[name = string("op_3792_transpose_x_1"), val = bool(true)];
+            bool var_3792_transpose_y_1 = const()[name = string("op_3792_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3792_cast_fp16 = matmul(transpose_x = var_3792_transpose_x_1, transpose_y = var_3792_transpose_y_1, x = var_3790_cast_fp16, y = v_new_147_cast_fp16)[name = string("op_3792_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_149_cast_fp16 = add(x = var_3788_cast_fp16, y = var_3792_cast_fp16)[name = string("state_149_cast_fp16")];
+            tensor<int32, [4]> var_3795_begin_0 = const()[name = string("op_3795_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_3795_end_0 = const()[name = string("op_3795_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_3795_end_mask_0 = const()[name = string("op_3795_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3795_squeeze_mask_0 = const()[name = string("op_3795_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3795_cast_fp16 = slice_by_index(begin = var_3795_begin_0, end = var_3795_end_0, end_mask = var_3795_end_mask_0, squeeze_mask = var_3795_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_3795_cast_fp16")];
+            tensor<int32, [4]> var_3797_begin_0 = const()[name = string("op_3797_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_3797_end_0 = const()[name = string("op_3797_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_3797_end_mask_0 = const()[name = string("op_3797_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3797_squeeze_mask_0 = const()[name = string("op_3797_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3797_cast_fp16 = slice_by_index(begin = var_3797_begin_0, end = var_3797_end_0, end_mask = var_3797_end_mask_0, squeeze_mask = var_3797_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_3797_cast_fp16")];
+            bool var_3798_transpose_x_0 = const()[name = string("op_3798_transpose_x_0"), val = bool(false)];
+            bool var_3798_transpose_y_0 = const()[name = string("op_3798_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3798_cast_fp16 = matmul(transpose_x = var_3798_transpose_x_0, transpose_y = var_3798_transpose_y_0, x = var_3797_cast_fp16, y = state_149_cast_fp16)[name = string("op_3798_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_149_cast_fp16 = sub(x = var_3795_cast_fp16, y = var_3798_cast_fp16)[name = string("v_new_149_cast_fp16")];
+            tensor<int32, [4]> var_3801_begin_0 = const()[name = string("op_3801_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_3801_end_0 = const()[name = string("op_3801_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_3801_end_mask_0 = const()[name = string("op_3801_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3801_squeeze_mask_0 = const()[name = string("op_3801_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3801_cast_fp16 = slice_by_index(begin = var_3801_begin_0, end = var_3801_end_0, end_mask = var_3801_end_mask_0, squeeze_mask = var_3801_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_3801_cast_fp16")];
+            bool var_3802_transpose_x_0 = const()[name = string("op_3802_transpose_x_0"), val = bool(false)];
+            bool var_3802_transpose_y_0 = const()[name = string("op_3802_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3802_cast_fp16 = matmul(transpose_x = var_3802_transpose_x_0, transpose_y = var_3802_transpose_y_0, x = var_3801_cast_fp16, y = state_149_cast_fp16)[name = string("op_3802_cast_fp16")];
+            tensor<int32, [4]> var_3804_begin_0 = const()[name = string("op_3804_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_3804_end_0 = const()[name = string("op_3804_end_0"), val = tensor<int32, [4]>([32, 11, 64, 64])];
+            tensor<bool, [4]> var_3804_end_mask_0 = const()[name = string("op_3804_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3804_squeeze_mask_0 = const()[name = string("op_3804_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3804_cast_fp16 = slice_by_index(begin = var_3804_begin_0, end = var_3804_end_0, end_mask = var_3804_end_mask_0, squeeze_mask = var_3804_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_3804_cast_fp16")];
+            bool var_3805_transpose_x_0 = const()[name = string("op_3805_transpose_x_0"), val = bool(false)];
+            bool var_3805_transpose_y_0 = const()[name = string("op_3805_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3805_cast_fp16 = matmul(transpose_x = var_3805_transpose_x_0, transpose_y = var_3805_transpose_y_0, x = var_3804_cast_fp16, y = v_new_149_cast_fp16)[name = string("op_3805_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3806_cast_fp16 = add(x = var_3802_cast_fp16, y = var_3805_cast_fp16)[name = string("op_3806_cast_fp16")];
+            tensor<int32, [4]> var_3808_begin_0 = const()[name = string("op_3808_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_3808_end_0 = const()[name = string("op_3808_end_0"), val = tensor<int32, [4]>([32, 11, 1, 1])];
+            tensor<bool, [4]> var_3808_end_mask_0 = const()[name = string("op_3808_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3808_squeeze_mask_0 = const()[name = string("op_3808_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3808_cast_fp16 = slice_by_index(begin = var_3808_begin_0, end = var_3808_end_0, end_mask = var_3808_end_mask_0, squeeze_mask = var_3808_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_3808_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3809_cast_fp16 = mul(x = state_149_cast_fp16, y = var_3808_cast_fp16)[name = string("op_3809_cast_fp16")];
+            tensor<int32, [4]> var_3811_begin_0 = const()[name = string("op_3811_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_3811_end_0 = const()[name = string("op_3811_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_3811_end_mask_0 = const()[name = string("op_3811_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3811_squeeze_mask_0 = const()[name = string("op_3811_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3811_cast_fp16 = slice_by_index(begin = var_3811_begin_0, end = var_3811_end_0, end_mask = var_3811_end_mask_0, squeeze_mask = var_3811_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_3811_cast_fp16")];
+            bool var_3813_transpose_x_1 = const()[name = string("op_3813_transpose_x_1"), val = bool(true)];
+            bool var_3813_transpose_y_1 = const()[name = string("op_3813_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3813_cast_fp16 = matmul(transpose_x = var_3813_transpose_x_1, transpose_y = var_3813_transpose_y_1, x = var_3811_cast_fp16, y = v_new_149_cast_fp16)[name = string("op_3813_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_151_cast_fp16 = add(x = var_3809_cast_fp16, y = var_3813_cast_fp16)[name = string("state_151_cast_fp16")];
+            tensor<int32, [4]> var_3816_begin_0 = const()[name = string("op_3816_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_3816_end_0 = const()[name = string("op_3816_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_3816_end_mask_0 = const()[name = string("op_3816_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3816_squeeze_mask_0 = const()[name = string("op_3816_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3816_cast_fp16 = slice_by_index(begin = var_3816_begin_0, end = var_3816_end_0, end_mask = var_3816_end_mask_0, squeeze_mask = var_3816_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_3816_cast_fp16")];
+            tensor<int32, [4]> var_3818_begin_0 = const()[name = string("op_3818_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_3818_end_0 = const()[name = string("op_3818_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_3818_end_mask_0 = const()[name = string("op_3818_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3818_squeeze_mask_0 = const()[name = string("op_3818_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3818_cast_fp16 = slice_by_index(begin = var_3818_begin_0, end = var_3818_end_0, end_mask = var_3818_end_mask_0, squeeze_mask = var_3818_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_3818_cast_fp16")];
+            bool var_3819_transpose_x_0 = const()[name = string("op_3819_transpose_x_0"), val = bool(false)];
+            bool var_3819_transpose_y_0 = const()[name = string("op_3819_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3819_cast_fp16 = matmul(transpose_x = var_3819_transpose_x_0, transpose_y = var_3819_transpose_y_0, x = var_3818_cast_fp16, y = state_151_cast_fp16)[name = string("op_3819_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_151_cast_fp16 = sub(x = var_3816_cast_fp16, y = var_3819_cast_fp16)[name = string("v_new_151_cast_fp16")];
+            tensor<int32, [4]> var_3822_begin_0 = const()[name = string("op_3822_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_3822_end_0 = const()[name = string("op_3822_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_3822_end_mask_0 = const()[name = string("op_3822_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3822_squeeze_mask_0 = const()[name = string("op_3822_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3822_cast_fp16 = slice_by_index(begin = var_3822_begin_0, end = var_3822_end_0, end_mask = var_3822_end_mask_0, squeeze_mask = var_3822_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_3822_cast_fp16")];
+            bool var_3823_transpose_x_0 = const()[name = string("op_3823_transpose_x_0"), val = bool(false)];
+            bool var_3823_transpose_y_0 = const()[name = string("op_3823_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3823_cast_fp16 = matmul(transpose_x = var_3823_transpose_x_0, transpose_y = var_3823_transpose_y_0, x = var_3822_cast_fp16, y = state_151_cast_fp16)[name = string("op_3823_cast_fp16")];
+            tensor<int32, [4]> var_3825_begin_0 = const()[name = string("op_3825_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_3825_end_0 = const()[name = string("op_3825_end_0"), val = tensor<int32, [4]>([32, 12, 64, 64])];
+            tensor<bool, [4]> var_3825_end_mask_0 = const()[name = string("op_3825_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3825_squeeze_mask_0 = const()[name = string("op_3825_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3825_cast_fp16 = slice_by_index(begin = var_3825_begin_0, end = var_3825_end_0, end_mask = var_3825_end_mask_0, squeeze_mask = var_3825_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_3825_cast_fp16")];
+            bool var_3826_transpose_x_0 = const()[name = string("op_3826_transpose_x_0"), val = bool(false)];
+            bool var_3826_transpose_y_0 = const()[name = string("op_3826_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3826_cast_fp16 = matmul(transpose_x = var_3826_transpose_x_0, transpose_y = var_3826_transpose_y_0, x = var_3825_cast_fp16, y = v_new_151_cast_fp16)[name = string("op_3826_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3827_cast_fp16 = add(x = var_3823_cast_fp16, y = var_3826_cast_fp16)[name = string("op_3827_cast_fp16")];
+            tensor<int32, [4]> var_3829_begin_0 = const()[name = string("op_3829_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_3829_end_0 = const()[name = string("op_3829_end_0"), val = tensor<int32, [4]>([32, 12, 1, 1])];
+            tensor<bool, [4]> var_3829_end_mask_0 = const()[name = string("op_3829_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3829_squeeze_mask_0 = const()[name = string("op_3829_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3829_cast_fp16 = slice_by_index(begin = var_3829_begin_0, end = var_3829_end_0, end_mask = var_3829_end_mask_0, squeeze_mask = var_3829_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_3829_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3830_cast_fp16 = mul(x = state_151_cast_fp16, y = var_3829_cast_fp16)[name = string("op_3830_cast_fp16")];
+            tensor<int32, [4]> var_3832_begin_0 = const()[name = string("op_3832_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_3832_end_0 = const()[name = string("op_3832_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_3832_end_mask_0 = const()[name = string("op_3832_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3832_squeeze_mask_0 = const()[name = string("op_3832_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3832_cast_fp16 = slice_by_index(begin = var_3832_begin_0, end = var_3832_end_0, end_mask = var_3832_end_mask_0, squeeze_mask = var_3832_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_3832_cast_fp16")];
+            bool var_3834_transpose_x_1 = const()[name = string("op_3834_transpose_x_1"), val = bool(true)];
+            bool var_3834_transpose_y_1 = const()[name = string("op_3834_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3834_cast_fp16 = matmul(transpose_x = var_3834_transpose_x_1, transpose_y = var_3834_transpose_y_1, x = var_3832_cast_fp16, y = v_new_151_cast_fp16)[name = string("op_3834_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_153_cast_fp16 = add(x = var_3830_cast_fp16, y = var_3834_cast_fp16)[name = string("state_153_cast_fp16")];
+            tensor<int32, [4]> var_3837_begin_0 = const()[name = string("op_3837_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_3837_end_0 = const()[name = string("op_3837_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_3837_end_mask_0 = const()[name = string("op_3837_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3837_squeeze_mask_0 = const()[name = string("op_3837_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3837_cast_fp16 = slice_by_index(begin = var_3837_begin_0, end = var_3837_end_0, end_mask = var_3837_end_mask_0, squeeze_mask = var_3837_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_3837_cast_fp16")];
+            tensor<int32, [4]> var_3839_begin_0 = const()[name = string("op_3839_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_3839_end_0 = const()[name = string("op_3839_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_3839_end_mask_0 = const()[name = string("op_3839_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3839_squeeze_mask_0 = const()[name = string("op_3839_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3839_cast_fp16 = slice_by_index(begin = var_3839_begin_0, end = var_3839_end_0, end_mask = var_3839_end_mask_0, squeeze_mask = var_3839_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_3839_cast_fp16")];
+            bool var_3840_transpose_x_0 = const()[name = string("op_3840_transpose_x_0"), val = bool(false)];
+            bool var_3840_transpose_y_0 = const()[name = string("op_3840_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3840_cast_fp16 = matmul(transpose_x = var_3840_transpose_x_0, transpose_y = var_3840_transpose_y_0, x = var_3839_cast_fp16, y = state_153_cast_fp16)[name = string("op_3840_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_153_cast_fp16 = sub(x = var_3837_cast_fp16, y = var_3840_cast_fp16)[name = string("v_new_153_cast_fp16")];
+            tensor<int32, [4]> var_3843_begin_0 = const()[name = string("op_3843_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_3843_end_0 = const()[name = string("op_3843_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_3843_end_mask_0 = const()[name = string("op_3843_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3843_squeeze_mask_0 = const()[name = string("op_3843_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3843_cast_fp16 = slice_by_index(begin = var_3843_begin_0, end = var_3843_end_0, end_mask = var_3843_end_mask_0, squeeze_mask = var_3843_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_3843_cast_fp16")];
+            bool var_3844_transpose_x_0 = const()[name = string("op_3844_transpose_x_0"), val = bool(false)];
+            bool var_3844_transpose_y_0 = const()[name = string("op_3844_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3844_cast_fp16 = matmul(transpose_x = var_3844_transpose_x_0, transpose_y = var_3844_transpose_y_0, x = var_3843_cast_fp16, y = state_153_cast_fp16)[name = string("op_3844_cast_fp16")];
+            tensor<int32, [4]> var_3846_begin_0 = const()[name = string("op_3846_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_3846_end_0 = const()[name = string("op_3846_end_0"), val = tensor<int32, [4]>([32, 13, 64, 64])];
+            tensor<bool, [4]> var_3846_end_mask_0 = const()[name = string("op_3846_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3846_squeeze_mask_0 = const()[name = string("op_3846_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3846_cast_fp16 = slice_by_index(begin = var_3846_begin_0, end = var_3846_end_0, end_mask = var_3846_end_mask_0, squeeze_mask = var_3846_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_3846_cast_fp16")];
+            bool var_3847_transpose_x_0 = const()[name = string("op_3847_transpose_x_0"), val = bool(false)];
+            bool var_3847_transpose_y_0 = const()[name = string("op_3847_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3847_cast_fp16 = matmul(transpose_x = var_3847_transpose_x_0, transpose_y = var_3847_transpose_y_0, x = var_3846_cast_fp16, y = v_new_153_cast_fp16)[name = string("op_3847_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3848_cast_fp16 = add(x = var_3844_cast_fp16, y = var_3847_cast_fp16)[name = string("op_3848_cast_fp16")];
+            tensor<int32, [4]> var_3850_begin_0 = const()[name = string("op_3850_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_3850_end_0 = const()[name = string("op_3850_end_0"), val = tensor<int32, [4]>([32, 13, 1, 1])];
+            tensor<bool, [4]> var_3850_end_mask_0 = const()[name = string("op_3850_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3850_squeeze_mask_0 = const()[name = string("op_3850_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3850_cast_fp16 = slice_by_index(begin = var_3850_begin_0, end = var_3850_end_0, end_mask = var_3850_end_mask_0, squeeze_mask = var_3850_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_3850_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3851_cast_fp16 = mul(x = state_153_cast_fp16, y = var_3850_cast_fp16)[name = string("op_3851_cast_fp16")];
+            tensor<int32, [4]> var_3853_begin_0 = const()[name = string("op_3853_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_3853_end_0 = const()[name = string("op_3853_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_3853_end_mask_0 = const()[name = string("op_3853_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3853_squeeze_mask_0 = const()[name = string("op_3853_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3853_cast_fp16 = slice_by_index(begin = var_3853_begin_0, end = var_3853_end_0, end_mask = var_3853_end_mask_0, squeeze_mask = var_3853_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_3853_cast_fp16")];
+            bool var_3855_transpose_x_1 = const()[name = string("op_3855_transpose_x_1"), val = bool(true)];
+            bool var_3855_transpose_y_1 = const()[name = string("op_3855_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3855_cast_fp16 = matmul(transpose_x = var_3855_transpose_x_1, transpose_y = var_3855_transpose_y_1, x = var_3853_cast_fp16, y = v_new_153_cast_fp16)[name = string("op_3855_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_155_cast_fp16 = add(x = var_3851_cast_fp16, y = var_3855_cast_fp16)[name = string("state_155_cast_fp16")];
+            tensor<int32, [4]> var_3858_begin_0 = const()[name = string("op_3858_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_3858_end_0 = const()[name = string("op_3858_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_3858_end_mask_0 = const()[name = string("op_3858_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3858_squeeze_mask_0 = const()[name = string("op_3858_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3858_cast_fp16 = slice_by_index(begin = var_3858_begin_0, end = var_3858_end_0, end_mask = var_3858_end_mask_0, squeeze_mask = var_3858_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_3858_cast_fp16")];
+            tensor<int32, [4]> var_3860_begin_0 = const()[name = string("op_3860_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_3860_end_0 = const()[name = string("op_3860_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_3860_end_mask_0 = const()[name = string("op_3860_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3860_squeeze_mask_0 = const()[name = string("op_3860_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3860_cast_fp16 = slice_by_index(begin = var_3860_begin_0, end = var_3860_end_0, end_mask = var_3860_end_mask_0, squeeze_mask = var_3860_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_3860_cast_fp16")];
+            bool var_3861_transpose_x_0 = const()[name = string("op_3861_transpose_x_0"), val = bool(false)];
+            bool var_3861_transpose_y_0 = const()[name = string("op_3861_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3861_cast_fp16 = matmul(transpose_x = var_3861_transpose_x_0, transpose_y = var_3861_transpose_y_0, x = var_3860_cast_fp16, y = state_155_cast_fp16)[name = string("op_3861_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_155_cast_fp16 = sub(x = var_3858_cast_fp16, y = var_3861_cast_fp16)[name = string("v_new_155_cast_fp16")];
+            tensor<int32, [4]> var_3864_begin_0 = const()[name = string("op_3864_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_3864_end_0 = const()[name = string("op_3864_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_3864_end_mask_0 = const()[name = string("op_3864_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3864_squeeze_mask_0 = const()[name = string("op_3864_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3864_cast_fp16 = slice_by_index(begin = var_3864_begin_0, end = var_3864_end_0, end_mask = var_3864_end_mask_0, squeeze_mask = var_3864_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_3864_cast_fp16")];
+            bool var_3865_transpose_x_0 = const()[name = string("op_3865_transpose_x_0"), val = bool(false)];
+            bool var_3865_transpose_y_0 = const()[name = string("op_3865_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3865_cast_fp16 = matmul(transpose_x = var_3865_transpose_x_0, transpose_y = var_3865_transpose_y_0, x = var_3864_cast_fp16, y = state_155_cast_fp16)[name = string("op_3865_cast_fp16")];
+            tensor<int32, [4]> var_3867_begin_0 = const()[name = string("op_3867_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_3867_end_0 = const()[name = string("op_3867_end_0"), val = tensor<int32, [4]>([32, 14, 64, 64])];
+            tensor<bool, [4]> var_3867_end_mask_0 = const()[name = string("op_3867_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3867_squeeze_mask_0 = const()[name = string("op_3867_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3867_cast_fp16 = slice_by_index(begin = var_3867_begin_0, end = var_3867_end_0, end_mask = var_3867_end_mask_0, squeeze_mask = var_3867_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_3867_cast_fp16")];
+            bool var_3868_transpose_x_0 = const()[name = string("op_3868_transpose_x_0"), val = bool(false)];
+            bool var_3868_transpose_y_0 = const()[name = string("op_3868_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3868_cast_fp16 = matmul(transpose_x = var_3868_transpose_x_0, transpose_y = var_3868_transpose_y_0, x = var_3867_cast_fp16, y = v_new_155_cast_fp16)[name = string("op_3868_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3869_cast_fp16 = add(x = var_3865_cast_fp16, y = var_3868_cast_fp16)[name = string("op_3869_cast_fp16")];
+            tensor<int32, [4]> var_3871_begin_0 = const()[name = string("op_3871_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_3871_end_0 = const()[name = string("op_3871_end_0"), val = tensor<int32, [4]>([32, 14, 1, 1])];
+            tensor<bool, [4]> var_3871_end_mask_0 = const()[name = string("op_3871_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3871_squeeze_mask_0 = const()[name = string("op_3871_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3871_cast_fp16 = slice_by_index(begin = var_3871_begin_0, end = var_3871_end_0, end_mask = var_3871_end_mask_0, squeeze_mask = var_3871_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_3871_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3872_cast_fp16 = mul(x = state_155_cast_fp16, y = var_3871_cast_fp16)[name = string("op_3872_cast_fp16")];
+            tensor<int32, [4]> var_3874_begin_0 = const()[name = string("op_3874_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_3874_end_0 = const()[name = string("op_3874_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_3874_end_mask_0 = const()[name = string("op_3874_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3874_squeeze_mask_0 = const()[name = string("op_3874_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3874_cast_fp16 = slice_by_index(begin = var_3874_begin_0, end = var_3874_end_0, end_mask = var_3874_end_mask_0, squeeze_mask = var_3874_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_3874_cast_fp16")];
+            bool var_3876_transpose_x_1 = const()[name = string("op_3876_transpose_x_1"), val = bool(true)];
+            bool var_3876_transpose_y_1 = const()[name = string("op_3876_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3876_cast_fp16 = matmul(transpose_x = var_3876_transpose_x_1, transpose_y = var_3876_transpose_y_1, x = var_3874_cast_fp16, y = v_new_155_cast_fp16)[name = string("op_3876_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_157_cast_fp16 = add(x = var_3872_cast_fp16, y = var_3876_cast_fp16)[name = string("state_157_cast_fp16")];
+            tensor<int32, [4]> var_3879_begin_0 = const()[name = string("op_3879_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_3879_end_0 = const()[name = string("op_3879_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_3879_end_mask_0 = const()[name = string("op_3879_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3879_squeeze_mask_0 = const()[name = string("op_3879_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3879_cast_fp16 = slice_by_index(begin = var_3879_begin_0, end = var_3879_end_0, end_mask = var_3879_end_mask_0, squeeze_mask = var_3879_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_3879_cast_fp16")];
+            tensor<int32, [4]> var_3881_begin_0 = const()[name = string("op_3881_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_3881_end_0 = const()[name = string("op_3881_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_3881_end_mask_0 = const()[name = string("op_3881_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3881_squeeze_mask_0 = const()[name = string("op_3881_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3881_cast_fp16 = slice_by_index(begin = var_3881_begin_0, end = var_3881_end_0, end_mask = var_3881_end_mask_0, squeeze_mask = var_3881_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_3881_cast_fp16")];
+            bool var_3882_transpose_x_0 = const()[name = string("op_3882_transpose_x_0"), val = bool(false)];
+            bool var_3882_transpose_y_0 = const()[name = string("op_3882_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3882_cast_fp16 = matmul(transpose_x = var_3882_transpose_x_0, transpose_y = var_3882_transpose_y_0, x = var_3881_cast_fp16, y = state_157_cast_fp16)[name = string("op_3882_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_157_cast_fp16 = sub(x = var_3879_cast_fp16, y = var_3882_cast_fp16)[name = string("v_new_157_cast_fp16")];
+            tensor<int32, [4]> var_3885_begin_0 = const()[name = string("op_3885_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_3885_end_0 = const()[name = string("op_3885_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_3885_end_mask_0 = const()[name = string("op_3885_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3885_squeeze_mask_0 = const()[name = string("op_3885_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3885_cast_fp16 = slice_by_index(begin = var_3885_begin_0, end = var_3885_end_0, end_mask = var_3885_end_mask_0, squeeze_mask = var_3885_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_3885_cast_fp16")];
+            bool var_3886_transpose_x_0 = const()[name = string("op_3886_transpose_x_0"), val = bool(false)];
+            bool var_3886_transpose_y_0 = const()[name = string("op_3886_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3886_cast_fp16 = matmul(transpose_x = var_3886_transpose_x_0, transpose_y = var_3886_transpose_y_0, x = var_3885_cast_fp16, y = state_157_cast_fp16)[name = string("op_3886_cast_fp16")];
+            tensor<int32, [4]> var_3888_begin_0 = const()[name = string("op_3888_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_3888_end_0 = const()[name = string("op_3888_end_0"), val = tensor<int32, [4]>([32, 15, 64, 64])];
+            tensor<bool, [4]> var_3888_end_mask_0 = const()[name = string("op_3888_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3888_squeeze_mask_0 = const()[name = string("op_3888_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3888_cast_fp16 = slice_by_index(begin = var_3888_begin_0, end = var_3888_end_0, end_mask = var_3888_end_mask_0, squeeze_mask = var_3888_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_3888_cast_fp16")];
+            bool var_3889_transpose_x_0 = const()[name = string("op_3889_transpose_x_0"), val = bool(false)];
+            bool var_3889_transpose_y_0 = const()[name = string("op_3889_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3889_cast_fp16 = matmul(transpose_x = var_3889_transpose_x_0, transpose_y = var_3889_transpose_y_0, x = var_3888_cast_fp16, y = v_new_157_cast_fp16)[name = string("op_3889_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3890_cast_fp16 = add(x = var_3886_cast_fp16, y = var_3889_cast_fp16)[name = string("op_3890_cast_fp16")];
+            tensor<int32, [4]> var_3892_begin_0 = const()[name = string("op_3892_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_3892_end_0 = const()[name = string("op_3892_end_0"), val = tensor<int32, [4]>([32, 15, 1, 1])];
+            tensor<bool, [4]> var_3892_end_mask_0 = const()[name = string("op_3892_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3892_squeeze_mask_0 = const()[name = string("op_3892_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3892_cast_fp16 = slice_by_index(begin = var_3892_begin_0, end = var_3892_end_0, end_mask = var_3892_end_mask_0, squeeze_mask = var_3892_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_3892_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3893_cast_fp16 = mul(x = state_157_cast_fp16, y = var_3892_cast_fp16)[name = string("op_3893_cast_fp16")];
+            tensor<int32, [4]> var_3895_begin_0 = const()[name = string("op_3895_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_3895_end_0 = const()[name = string("op_3895_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_3895_end_mask_0 = const()[name = string("op_3895_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3895_squeeze_mask_0 = const()[name = string("op_3895_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3895_cast_fp16 = slice_by_index(begin = var_3895_begin_0, end = var_3895_end_0, end_mask = var_3895_end_mask_0, squeeze_mask = var_3895_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_3895_cast_fp16")];
+            bool var_3897_transpose_x_1 = const()[name = string("op_3897_transpose_x_1"), val = bool(true)];
+            bool var_3897_transpose_y_1 = const()[name = string("op_3897_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3897_cast_fp16 = matmul(transpose_x = var_3897_transpose_x_1, transpose_y = var_3897_transpose_y_1, x = var_3895_cast_fp16, y = v_new_157_cast_fp16)[name = string("op_3897_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_159_cast_fp16 = add(x = var_3893_cast_fp16, y = var_3897_cast_fp16)[name = string("state_159_cast_fp16")];
+            tensor<int32, [4]> var_3900_begin_0 = const()[name = string("op_3900_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_3900_end_0 = const()[name = string("op_3900_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_3900_end_mask_0 = const()[name = string("op_3900_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3900_squeeze_mask_0 = const()[name = string("op_3900_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3900_cast_fp16 = slice_by_index(begin = var_3900_begin_0, end = var_3900_end_0, end_mask = var_3900_end_mask_0, squeeze_mask = var_3900_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_3900_cast_fp16")];
+            tensor<int32, [4]> var_3902_begin_0 = const()[name = string("op_3902_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_3902_end_0 = const()[name = string("op_3902_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_3902_end_mask_0 = const()[name = string("op_3902_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3902_squeeze_mask_0 = const()[name = string("op_3902_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3902_cast_fp16 = slice_by_index(begin = var_3902_begin_0, end = var_3902_end_0, end_mask = var_3902_end_mask_0, squeeze_mask = var_3902_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_3902_cast_fp16")];
+            bool var_3903_transpose_x_0 = const()[name = string("op_3903_transpose_x_0"), val = bool(false)];
+            bool var_3903_transpose_y_0 = const()[name = string("op_3903_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3903_cast_fp16 = matmul(transpose_x = var_3903_transpose_x_0, transpose_y = var_3903_transpose_y_0, x = var_3902_cast_fp16, y = state_159_cast_fp16)[name = string("op_3903_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_159_cast_fp16 = sub(x = var_3900_cast_fp16, y = var_3903_cast_fp16)[name = string("v_new_159_cast_fp16")];
+            tensor<int32, [4]> var_3906_begin_0 = const()[name = string("op_3906_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_3906_end_0 = const()[name = string("op_3906_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_3906_end_mask_0 = const()[name = string("op_3906_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3906_squeeze_mask_0 = const()[name = string("op_3906_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3906_cast_fp16 = slice_by_index(begin = var_3906_begin_0, end = var_3906_end_0, end_mask = var_3906_end_mask_0, squeeze_mask = var_3906_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_3906_cast_fp16")];
+            bool var_3907_transpose_x_0 = const()[name = string("op_3907_transpose_x_0"), val = bool(false)];
+            bool var_3907_transpose_y_0 = const()[name = string("op_3907_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3907_cast_fp16 = matmul(transpose_x = var_3907_transpose_x_0, transpose_y = var_3907_transpose_y_0, x = var_3906_cast_fp16, y = state_159_cast_fp16)[name = string("op_3907_cast_fp16")];
+            tensor<int32, [4]> var_3909_begin_0 = const()[name = string("op_3909_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_3909_end_0 = const()[name = string("op_3909_end_0"), val = tensor<int32, [4]>([32, 16, 64, 64])];
+            tensor<bool, [4]> var_3909_end_mask_0 = const()[name = string("op_3909_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3909_squeeze_mask_0 = const()[name = string("op_3909_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3909_cast_fp16 = slice_by_index(begin = var_3909_begin_0, end = var_3909_end_0, end_mask = var_3909_end_mask_0, squeeze_mask = var_3909_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_3909_cast_fp16")];
+            bool var_3910_transpose_x_0 = const()[name = string("op_3910_transpose_x_0"), val = bool(false)];
+            bool var_3910_transpose_y_0 = const()[name = string("op_3910_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3910_cast_fp16 = matmul(transpose_x = var_3910_transpose_x_0, transpose_y = var_3910_transpose_y_0, x = var_3909_cast_fp16, y = v_new_159_cast_fp16)[name = string("op_3910_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3911_cast_fp16 = add(x = var_3907_cast_fp16, y = var_3910_cast_fp16)[name = string("op_3911_cast_fp16")];
+            int32 var_3913_axis_0 = const()[name = string("op_3913_axis_0"), val = int32(1)];
+            tensor<fp16, [32, 16, 64, 128]> var_3913_cast_fp16 = stack(axis = var_3913_axis_0, values = (var_3596_cast_fp16, var_3617_cast_fp16, var_3638_cast_fp16, var_3659_cast_fp16, var_3680_cast_fp16, var_3701_cast_fp16, var_3722_cast_fp16, var_3743_cast_fp16, var_3764_cast_fp16, var_3785_cast_fp16, var_3806_cast_fp16, var_3827_cast_fp16, var_3848_cast_fp16, var_3869_cast_fp16, var_3890_cast_fp16, var_3911_cast_fp16))[name = string("op_3913_cast_fp16")];
+            tensor<int32, [3]> var_3914 = const()[name = string("op_3914"), val = tensor<int32, [3]>([32, 1024, 128])];
+            tensor<fp16, [32, 1024, 128]> var_3915_cast_fp16 = reshape(shape = var_3914, x = var_3913_cast_fp16)[name = string("op_3915_cast_fp16")];
+            tensor<int32, [3]> var_3916 = const()[name = string("op_3916"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> input_79_begin_0 = const()[name = string("input_79_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> input_79_end_0 = const()[name = string("input_79_end_0"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<bool, [4]> input_79_end_mask_0 = const()[name = string("input_79_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> input_79_squeeze_mask_0 = const()[name = string("input_79_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [1024, 32, 128]> input_79_cast_fp16 = slice_by_index(begin = input_79_begin_0, end = input_79_end_0, end_mask = input_79_end_mask_0, squeeze_mask = input_79_squeeze_mask_0, x = z_9_cast_fp16)[name = string("input_79_cast_fp16")];
+            fp16 var_3232_promoted_1_to_fp16 = const()[name = string("op_3232_promoted_1_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1024, 32, 128]> x_143_cast_fp16 = transpose(perm = var_3916, x = var_3915_cast_fp16)[name = string("transpose_12")];
+            tensor<fp16, [1024, 32, 128]> var_3920_cast_fp16 = pow(x = x_143_cast_fp16, y = var_3232_promoted_1_to_fp16)[name = string("op_3920_cast_fp16")];
+            tensor<int32, [1]> var_3922_axes_0 = const()[name = string("op_3922_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_3922_keep_dims_0 = const()[name = string("op_3922_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1024, 32, 1]> var_3922_cast_fp16 = reduce_mean(axes = var_3922_axes_0, keep_dims = var_3922_keep_dims_0, x = var_3920_cast_fp16)[name = string("op_3922_cast_fp16")];
+            fp16 var_3923_to_fp16 = const()[name = string("op_3923_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1024, 32, 1]> var_3924_cast_fp16 = add(x = var_3922_cast_fp16, y = var_3923_to_fp16)[name = string("op_3924_cast_fp16")];
+            fp32 var_3925_epsilon_0 = const()[name = string("op_3925_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1024, 32, 1]> var_3925_cast_fp16 = rsqrt(epsilon = var_3925_epsilon_0, x = var_3924_cast_fp16)[name = string("op_3925_cast_fp16")];
+            tensor<fp16, [1024, 32, 128]> x_145_cast_fp16 = mul(x = x_143_cast_fp16, y = var_3925_cast_fp16)[name = string("x_145_cast_fp16")];
+            tensor<fp16, [128]> layers_5_linear_attn_norm_weight_to_fp16 = const()[name = string("layers_5_linear_attn_norm_weight_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(687944896)))];
+            tensor<fp16, [1024, 32, 128]> var_3927_cast_fp16 = mul(x = layers_5_linear_attn_norm_weight_to_fp16, y = x_145_cast_fp16)[name = string("op_3927_cast_fp16")];
+            tensor<fp16, [1024, 32, 128]> var_3928_cast_fp16 = silu(x = input_79_cast_fp16)[name = string("op_3928_cast_fp16")];
+            tensor<fp16, [1024, 32, 128]> core_9_cast_fp16 = mul(x = var_3927_cast_fp16, y = var_3928_cast_fp16)[name = string("core_9_cast_fp16")];
+            tensor<int32, [3]> var_3930 = const()[name = string("op_3930"), val = tensor<int32, [3]>([1, 1024, 4096])];
+            tensor<fp16, [1, 1024, 4096]> input_81_cast_fp16 = reshape(shape = var_3930, x = core_9_cast_fp16)[name = string("input_81_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> linear_43_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers_5_linear_attn_out_proj_weight_cast_fp16, x = input_81_cast_fp16)[name = string("linear_43_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> x_147_cast_fp16 = add(x = x_125_cast_fp16, y = linear_43_cast_fp16)[name = string("x_147_cast_fp16")];
+            fp16 var_3232_promoted_2_to_fp16 = const()[name = string("op_3232_promoted_2_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 1024, 2560]> var_3936_cast_fp16 = pow(x = x_147_cast_fp16, y = var_3232_promoted_2_to_fp16)[name = string("op_3936_cast_fp16")];
+            tensor<int32, [1]> var_3938_axes_0 = const()[name = string("op_3938_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_3938_keep_dims_0 = const()[name = string("op_3938_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 1024, 1]> var_3938_cast_fp16 = reduce_mean(axes = var_3938_axes_0, keep_dims = var_3938_keep_dims_0, x = var_3936_cast_fp16)[name = string("op_3938_cast_fp16")];
+            fp16 var_3939_to_fp16 = const()[name = string("op_3939_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 1024, 1]> var_3940_cast_fp16 = add(x = var_3938_cast_fp16, y = var_3939_to_fp16)[name = string("op_3940_cast_fp16")];
+            fp32 var_3941_epsilon_0 = const()[name = string("op_3941_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 1024, 1]> var_3941_cast_fp16 = rsqrt(epsilon = var_3941_epsilon_0, x = var_3940_cast_fp16)[name = string("op_3941_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> x_149_cast_fp16 = mul(x = x_147_cast_fp16, y = var_3941_cast_fp16)[name = string("x_149_cast_fp16")];
+            tensor<fp16, [2560]> var_3944_to_fp16 = const()[name = string("op_3944_to_fp16"), val = tensor<fp16, [2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(687945216)))];
+            tensor<fp16, [1, 1024, 2560]> input_83_cast_fp16 = mul(x = x_149_cast_fp16, y = var_3944_to_fp16)[name = string("input_83_cast_fp16")];
+            tensor<fp16, [1, 1024, 9216]> linear_44_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = layers_5_mlp_gate_proj_weight_cast_fp16, x = input_83_cast_fp16)[name = string("linear_44_cast_fp16")];
+            tensor<fp16, [1, 1024, 9216]> var_3951_cast_fp16 = silu(x = linear_44_cast_fp16)[name = string("op_3951_cast_fp16")];
+            tensor<fp16, [1, 1024, 9216]> linear_45_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = layers_5_mlp_up_proj_weight_cast_fp16, x = input_83_cast_fp16)[name = string("linear_45_cast_fp16")];
+            tensor<fp16, [1, 1024, 9216]> input_87_cast_fp16 = mul(x = var_3951_cast_fp16, y = linear_45_cast_fp16)[name = string("input_87_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> linear_46_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers_5_mlp_down_proj_weight_cast_fp16, x = input_87_cast_fp16)[name = string("linear_46_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> x_151_cast_fp16 = add(x = x_147_cast_fp16, y = linear_46_cast_fp16)[name = string("x_151_cast_fp16")];
+            int32 var_3971 = const()[name = string("op_3971"), val = int32(-2)];
+            int32 var_3986 = const()[name = string("op_3986"), val = int32(-1)];
+            fp16 var_3985_promoted_to_fp16 = const()[name = string("op_3985_promoted_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 1024, 2560]> var_3995_cast_fp16 = pow(x = x_151_cast_fp16, y = var_3985_promoted_to_fp16)[name = string("op_3995_cast_fp16")];
+            tensor<int32, [1]> var_3997_axes_0 = const()[name = string("op_3997_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_3997_keep_dims_0 = const()[name = string("op_3997_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 1024, 1]> var_3997_cast_fp16 = reduce_mean(axes = var_3997_axes_0, keep_dims = var_3997_keep_dims_0, x = var_3995_cast_fp16)[name = string("op_3997_cast_fp16")];
+            fp16 var_3998_to_fp16 = const()[name = string("op_3998_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 1024, 1]> var_3999_cast_fp16 = add(x = var_3997_cast_fp16, y = var_3998_to_fp16)[name = string("op_3999_cast_fp16")];
+            fp32 var_4000_epsilon_0 = const()[name = string("op_4000_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 1024, 1]> var_4000_cast_fp16 = rsqrt(epsilon = var_4000_epsilon_0, x = var_3999_cast_fp16)[name = string("op_4000_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> x_153_cast_fp16 = mul(x = x_151_cast_fp16, y = var_4000_cast_fp16)[name = string("x_153_cast_fp16")];
+            tensor<fp16, [2560]> var_4003_to_fp16 = const()[name = string("op_4003_to_fp16"), val = tensor<fp16, [2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(687950400)))];
+            tensor<fp16, [1, 1024, 2560]> x_155_cast_fp16 = mul(x = x_153_cast_fp16, y = var_4003_to_fp16)[name = string("x_155_cast_fp16")];
+            tensor<fp16, [1, 1024, 8192]> linear_47_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = layers_6_linear_attn_in_proj_qkv_weight_cast_fp16, x = x_155_cast_fp16)[name = string("linear_47_cast_fp16")];
+            tensor<int32, [3]> input_89_perm_0 = const()[name = string("input_89_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
+            tensor<int32, [6]> input_91_pad_0 = const()[name = string("input_91_pad_0"), val = tensor<int32, [6]>([0, 0, 0, 0, 3, 0])];
+            string input_91_mode_0 = const()[name = string("input_91_mode_0"), val = string("constant")];
+            fp16 const_66_to_fp16 = const()[name = string("const_66_to_fp16"), val = fp16(0x0p+0)];
+            tensor<fp16, [1, 8192, 1024]> input_89_cast_fp16 = transpose(perm = input_89_perm_0, x = linear_47_cast_fp16)[name = string("transpose_11")];
+            tensor<fp16, [1, 8192, 1027]> input_91_cast_fp16 = pad(constant_val = const_66_to_fp16, mode = input_91_mode_0, pad = input_91_pad_0, x = input_89_cast_fp16)[name = string("input_91_cast_fp16")];
+            string input_93_pad_type_0 = const()[name = string("input_93_pad_type_0"), val = string("valid")];
+            int32 input_93_groups_0 = const()[name = string("input_93_groups_0"), val = int32(8192)];
+            tensor<int32, [1]> input_93_strides_0 = const()[name = string("input_93_strides_0"), val = tensor<int32, [1]>([1])];
+            tensor<int32, [2]> input_93_pad_0 = const()[name = string("input_93_pad_0"), val = tensor<int32, [2]>([0, 0])];
+            tensor<int32, [1]> input_93_dilations_0 = const()[name = string("input_93_dilations_0"), val = tensor<int32, [1]>([1])];
+            tensor<fp16, [8192, 1, 4]> layers_6_linear_attn_conv1d_weight_to_fp16 = const()[name = string("layers_6_linear_attn_conv1d_weight_to_fp16"), val = tensor<fp16, [8192, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(687955584)))];
+            tensor<fp16, [1, 8192, 1024]> input_93_cast_fp16 = conv(dilations = input_93_dilations_0, groups = input_93_groups_0, pad = input_93_pad_0, pad_type = input_93_pad_type_0, strides = input_93_strides_0, weight = layers_6_linear_attn_conv1d_weight_to_fp16, x = input_91_cast_fp16)[name = string("input_93_cast_fp16")];
+            tensor<fp16, [1, 8192, 1024]> var_4038_cast_fp16 = silu(x = input_93_cast_fp16)[name = string("op_4038_cast_fp16")];
+            tensor<int32, [3]> qkv_perm_0 = const()[name = string("qkv_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
+            tensor<int32, [3]> var_4040_begin_0 = const()[name = string("op_4040_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_4040_end_0 = const()[name = string("op_4040_end_0"), val = tensor<int32, [3]>([1, 1024, 2048])];
+            tensor<bool, [3]> var_4040_end_mask_0 = const()[name = string("op_4040_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
+            tensor<fp16, [1, 1024, 8192]> qkv_cast_fp16 = transpose(perm = qkv_perm_0, x = var_4038_cast_fp16)[name = string("transpose_10")];
+            tensor<fp16, [1, 1024, 2048]> var_4040_cast_fp16 = slice_by_index(begin = var_4040_begin_0, end = var_4040_end_0, end_mask = var_4040_end_mask_0, x = qkv_cast_fp16)[name = string("op_4040_cast_fp16")];
+            tensor<int32, [4]> var_4041 = const()[name = string("op_4041"), val = tensor<int32, [4]>([1, 1024, 16, 128])];
+            tensor<fp16, [1, 1024, 16, 128]> q_55_cast_fp16 = reshape(shape = var_4041, x = var_4040_cast_fp16)[name = string("q_55_cast_fp16")];
+            tensor<int32, [3]> var_4043_begin_0 = const()[name = string("op_4043_begin_0"), val = tensor<int32, [3]>([0, 0, 2048])];
+            tensor<int32, [3]> var_4043_end_0 = const()[name = string("op_4043_end_0"), val = tensor<int32, [3]>([1, 1024, 4096])];
+            tensor<bool, [3]> var_4043_end_mask_0 = const()[name = string("op_4043_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
+            tensor<fp16, [1, 1024, 2048]> var_4043_cast_fp16 = slice_by_index(begin = var_4043_begin_0, end = var_4043_end_0, end_mask = var_4043_end_mask_0, x = qkv_cast_fp16)[name = string("op_4043_cast_fp16")];
+            tensor<int32, [4]> var_4044 = const()[name = string("op_4044"), val = tensor<int32, [4]>([1, 1024, 16, 128])];
+            tensor<fp16, [1, 1024, 16, 128]> k_47_cast_fp16 = reshape(shape = var_4044, x = var_4043_cast_fp16)[name = string("k_47_cast_fp16")];
+            tensor<int32, [3]> var_4046_begin_0 = const()[name = string("op_4046_begin_0"), val = tensor<int32, [3]>([0, 0, 4096])];
+            tensor<int32, [3]> var_4046_end_0 = const()[name = string("op_4046_end_0"), val = tensor<int32, [3]>([1, 1024, 8192])];
+            tensor<bool, [3]> var_4046_end_mask_0 = const()[name = string("op_4046_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
+            tensor<fp16, [1, 1024, 4096]> var_4046_cast_fp16 = slice_by_index(begin = var_4046_begin_0, end = var_4046_end_0, end_mask = var_4046_end_mask_0, x = qkv_cast_fp16)[name = string("op_4046_cast_fp16")];
+            tensor<int32, [4]> var_4047 = const()[name = string("op_4047"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<fp16, [1, 1024, 32, 128]> v_25_cast_fp16 = reshape(shape = var_4047, x = var_4046_cast_fp16)[name = string("v_25_cast_fp16")];
+            tensor<fp16, [1, 1024, 4096]> linear_48_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = layers_6_linear_attn_in_proj_z_weight_cast_fp16, x = x_155_cast_fp16)[name = string("linear_48_cast_fp16")];
+            tensor<int32, [4]> var_4051 = const()[name = string("op_4051"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<fp16, [1, 1024, 32, 128]> z_cast_fp16 = reshape(shape = var_4051, x = linear_48_cast_fp16)[name = string("z_cast_fp16")];
+            tensor<fp16, [32]> linear_49_bias_0_to_fp16 = const()[name = string("linear_49_bias_0_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(688021184)))];
+            tensor<fp16, [1, 1024, 32]> linear_49_cast_fp16 = linear(bias = linear_49_bias_0_to_fp16, weight = layers_6_linear_attn_in_proj_b_weight_cast_fp16, x = x_155_cast_fp16)[name = string("linear_49_cast_fp16")];
+            tensor<fp16, [1, 1024, 32]> beta_21_cast_fp16 = sigmoid(x = linear_49_cast_fp16)[name = string("beta_21_cast_fp16")];
+            tensor<fp16, [32]> linear_50_bias_0_to_fp16 = const()[name = string("linear_50_bias_0_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(688021312)))];
+            tensor<fp16, [1, 1024, 32]> linear_50_cast_fp16 = linear(bias = linear_50_bias_0_to_fp16, weight = layers_6_linear_attn_in_proj_a_weight_cast_fp16, x = x_155_cast_fp16)[name = string("linear_50_cast_fp16")];
+            tensor<fp16, [32]> layers_6_linear_attn_dt_bias_to_fp16 = const()[name = string("layers_6_linear_attn_dt_bias_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(688021440)))];
+            tensor<fp16, [1, 1024, 32]> var_4060_cast_fp16 = add(x = linear_50_cast_fp16, y = layers_6_linear_attn_dt_bias_to_fp16)[name = string("op_4060_cast_fp16")];
+            tensor<fp16, [1, 1024, 32]> var_4061_cast_fp16 = softplus(x = var_4060_cast_fp16)[name = string("op_4061_cast_fp16")];
+            tensor<fp16, [32]> var_4057_to_fp16 = const()[name = string("op_4057_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(688021568)))];
+            tensor<fp16, [1, 1024, 32]> g_21_cast_fp16 = mul(x = var_4057_to_fp16, y = var_4061_cast_fp16)[name = string("g_21_cast_fp16")];
+            tensor<fp16, [1, 1024, 16, 128]> var_4063_cast_fp16 = mul(x = q_55_cast_fp16, y = q_55_cast_fp16)[name = string("op_4063_cast_fp16")];
+            tensor<int32, [1]> var_4065_axes_0 = const()[name = string("op_4065_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_4065_keep_dims_0 = const()[name = string("op_4065_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 1024, 16, 1]> var_4065_cast_fp16 = reduce_sum(axes = var_4065_axes_0, keep_dims = var_4065_keep_dims_0, x = var_4063_cast_fp16)[name = string("op_4065_cast_fp16")];
+            fp16 var_4066_to_fp16 = const()[name = string("op_4066_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 1024, 16, 1]> var_4067_cast_fp16 = add(x = var_4065_cast_fp16, y = var_4066_to_fp16)[name = string("op_4067_cast_fp16")];
+            fp32 var_4068_epsilon_0 = const()[name = string("op_4068_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 1024, 16, 1]> var_4068_cast_fp16 = rsqrt(epsilon = var_4068_epsilon_0, x = var_4067_cast_fp16)[name = string("op_4068_cast_fp16")];
+            tensor<fp16, [1, 1024, 16, 128]> q_57_cast_fp16 = mul(x = q_55_cast_fp16, y = var_4068_cast_fp16)[name = string("q_57_cast_fp16")];
+            tensor<fp16, [1, 1024, 16, 128]> var_4070_cast_fp16 = mul(x = k_47_cast_fp16, y = k_47_cast_fp16)[name = string("op_4070_cast_fp16")];
+            tensor<int32, [1]> var_4072_axes_0 = const()[name = string("op_4072_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_4072_keep_dims_0 = const()[name = string("op_4072_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 1024, 16, 1]> var_4072_cast_fp16 = reduce_sum(axes = var_4072_axes_0, keep_dims = var_4072_keep_dims_0, x = var_4070_cast_fp16)[name = string("op_4072_cast_fp16")];
+            fp16 var_4073_to_fp16 = const()[name = string("op_4073_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 1024, 16, 1]> var_4074_cast_fp16 = add(x = var_4072_cast_fp16, y = var_4073_to_fp16)[name = string("op_4074_cast_fp16")];
+            fp32 var_4075_epsilon_0 = const()[name = string("op_4075_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 1024, 16, 1]> var_4075_cast_fp16 = rsqrt(epsilon = var_4075_epsilon_0, x = var_4074_cast_fp16)[name = string("op_4075_cast_fp16")];
+            tensor<fp16, [1, 1024, 16, 128]> k_49_cast_fp16 = mul(x = k_47_cast_fp16, y = var_4075_cast_fp16)[name = string("k_49_cast_fp16")];
+            tensor<int32, [1]> var_4080_axes_0 = const()[name = string("op_4080_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [1, 1024, 16, 1, 128]> var_4080_cast_fp16 = expand_dims(axes = var_4080_axes_0, x = q_57_cast_fp16)[name = string("op_4080_cast_fp16")];
+            tensor<int32, [5]> var_4082_reps_0 = const()[name = string("op_4082_reps_0"), val = tensor<int32, [5]>([1, 1, 1, 2, 1])];
+            tensor<fp16, [1, 1024, 16, 2, 128]> var_4082_cast_fp16 = tile(reps = var_4082_reps_0, x = var_4080_cast_fp16)[name = string("op_4082_cast_fp16")];
+            tensor<int32, [4]> var_4083 = const()[name = string("op_4083"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<fp16, [1, 1024, 32, 128]> q_59_cast_fp16 = reshape(shape = var_4083, x = var_4082_cast_fp16)[name = string("q_59_cast_fp16")];
+            tensor<int32, [1]> var_4088_axes_0 = const()[name = string("op_4088_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [1, 1024, 16, 1, 128]> var_4088_cast_fp16 = expand_dims(axes = var_4088_axes_0, x = k_49_cast_fp16)[name = string("op_4088_cast_fp16")];
+            tensor<int32, [5]> var_4090_reps_0 = const()[name = string("op_4090_reps_0"), val = tensor<int32, [5]>([1, 1, 1, 2, 1])];
+            tensor<fp16, [1, 1024, 16, 2, 128]> var_4090_cast_fp16 = tile(reps = var_4090_reps_0, x = var_4088_cast_fp16)[name = string("op_4090_cast_fp16")];
+            tensor<int32, [4]> var_4091 = const()[name = string("op_4091"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<fp16, [1, 1024, 32, 128]> k_51_cast_fp16 = reshape(shape = var_4091, x = var_4090_cast_fp16)[name = string("k_51_cast_fp16")];
+            fp16 var_4093_to_fp16 = const()[name = string("op_4093_to_fp16"), val = fp16(0x1.6ap-4)];
+            tensor<fp16, [1, 1024, 32, 128]> q_61_cast_fp16 = mul(x = q_59_cast_fp16, y = var_4093_to_fp16)[name = string("q_61_cast_fp16")];
+            tensor<int32, [4]> var_4095_begin_0 = const()[name = string("op_4095_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_4095_end_0 = const()[name = string("op_4095_end_0"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<bool, [4]> var_4095_end_mask_0 = const()[name = string("op_4095_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_4095_squeeze_mask_0 = const()[name = string("op_4095_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [1024, 32, 128]> var_4095_cast_fp16 = slice_by_index(begin = var_4095_begin_0, end = var_4095_end_0, end_mask = var_4095_end_mask_0, squeeze_mask = var_4095_squeeze_mask_0, x = q_61_cast_fp16)[name = string("op_4095_cast_fp16")];
+            tensor<int32, [3]> var_4096 = const()[name = string("op_4096"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_4098 = const()[name = string("op_4098"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<fp16, [32, 1024, 128]> var_4097_cast_fp16 = transpose(perm = var_4096, x = var_4095_cast_fp16)[name = string("transpose_9")];
+            tensor<fp16, [32, 16, 64, 128]> q_63_cast_fp16 = reshape(shape = var_4098, x = var_4097_cast_fp16)[name = string("q_63_cast_fp16")];
+            tensor<int32, [4]> var_4100_begin_0 = const()[name = string("op_4100_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_4100_end_0 = const()[name = string("op_4100_end_0"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<bool, [4]> var_4100_end_mask_0 = const()[name = string("op_4100_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_4100_squeeze_mask_0 = const()[name = string("op_4100_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [1024, 32, 128]> var_4100_cast_fp16 = slice_by_index(begin = var_4100_begin_0, end = var_4100_end_0, end_mask = var_4100_end_mask_0, squeeze_mask = var_4100_squeeze_mask_0, x = k_51_cast_fp16)[name = string("op_4100_cast_fp16")];
+            tensor<int32, [3]> var_4101 = const()[name = string("op_4101"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_4103 = const()[name = string("op_4103"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<fp16, [32, 1024, 128]> var_4102_cast_fp16 = transpose(perm = var_4101, x = var_4100_cast_fp16)[name = string("transpose_8")];
+            tensor<fp16, [32, 16, 64, 128]> k_53_cast_fp16 = reshape(shape = var_4103, x = var_4102_cast_fp16)[name = string("k_53_cast_fp16")];
+            tensor<int32, [4]> var_4105_begin_0 = const()[name = string("op_4105_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_4105_end_0 = const()[name = string("op_4105_end_0"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<bool, [4]> var_4105_end_mask_0 = const()[name = string("op_4105_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_4105_squeeze_mask_0 = const()[name = string("op_4105_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [1024, 32, 128]> var_4105_cast_fp16 = slice_by_index(begin = var_4105_begin_0, end = var_4105_end_0, end_mask = var_4105_end_mask_0, squeeze_mask = var_4105_squeeze_mask_0, x = v_25_cast_fp16)[name = string("op_4105_cast_fp16")];
+            tensor<int32, [3]> var_4106 = const()[name = string("op_4106"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_4108 = const()[name = string("op_4108"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<fp16, [32, 1024, 128]> var_4107_cast_fp16 = transpose(perm = var_4106, x = var_4105_cast_fp16)[name = string("transpose_7")];
+            tensor<fp16, [32, 16, 64, 128]> v_27_cast_fp16 = reshape(shape = var_4108, x = var_4107_cast_fp16)[name = string("v_27_cast_fp16")];
+            tensor<int32, [3]> var_4110_begin_0 = const()[name = string("op_4110_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_4110_end_0 = const()[name = string("op_4110_end_0"), val = tensor<int32, [3]>([1, 1024, 32])];
+            tensor<bool, [3]> var_4110_end_mask_0 = const()[name = string("op_4110_end_mask_0"), val = tensor<bool, [3]>([false, true, true])];
+            tensor<bool, [3]> var_4110_squeeze_mask_0 = const()[name = string("op_4110_squeeze_mask_0"), val = tensor<bool, [3]>([true, false, false])];
+            tensor<fp16, [1024, 32]> var_4110_cast_fp16 = slice_by_index(begin = var_4110_begin_0, end = var_4110_end_0, end_mask = var_4110_end_mask_0, squeeze_mask = var_4110_squeeze_mask_0, x = beta_21_cast_fp16)[name = string("op_4110_cast_fp16")];
+            tensor<int32, [2]> var_4111_perm_0 = const()[name = string("op_4111_perm_0"), val = tensor<int32, [2]>([1, 0])];
+            tensor<int32, [4]> var_4112 = const()[name = string("op_4112"), val = tensor<int32, [4]>([32, 16, 64, 1])];
+            tensor<fp16, [32, 1024]> var_4111_cast_fp16 = transpose(perm = var_4111_perm_0, x = var_4110_cast_fp16)[name = string("transpose_6")];
+            tensor<fp16, [32, 16, 64, 1]> beta_cast_fp16 = reshape(shape = var_4112, x = var_4111_cast_fp16)[name = string("beta_cast_fp16")];
+            tensor<int32, [3]> var_4114_begin_0 = const()[name = string("op_4114_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_4114_end_0 = const()[name = string("op_4114_end_0"), val = tensor<int32, [3]>([1, 1024, 32])];
+            tensor<bool, [3]> var_4114_end_mask_0 = const()[name = string("op_4114_end_mask_0"), val = tensor<bool, [3]>([false, true, true])];
+            tensor<bool, [3]> var_4114_squeeze_mask_0 = const()[name = string("op_4114_squeeze_mask_0"), val = tensor<bool, [3]>([true, false, false])];
+            tensor<fp16, [1024, 32]> var_4114_cast_fp16 = slice_by_index(begin = var_4114_begin_0, end = var_4114_end_0, end_mask = var_4114_end_mask_0, squeeze_mask = var_4114_squeeze_mask_0, x = g_21_cast_fp16)[name = string("op_4114_cast_fp16")];
+            tensor<int32, [2]> var_4115_perm_0 = const()[name = string("op_4115_perm_0"), val = tensor<int32, [2]>([1, 0])];
+            tensor<int32, [3]> var_4116 = const()[name = string("op_4116"), val = tensor<int32, [3]>([32, 16, 64])];
+            tensor<fp16, [32, 1024]> var_4115_cast_fp16 = transpose(perm = var_4115_perm_0, x = var_4114_cast_fp16)[name = string("transpose_5")];
+            tensor<fp16, [32, 16, 64]> g_cast_fp16 = reshape(shape = var_4116, x = var_4115_cast_fp16)[name = string("g_cast_fp16")];
+            tensor<int32, [1]> g_col_axes_0 = const()[name = string("g_col_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 16, 64, 1]> g_col_cast_fp16 = expand_dims(axes = g_col_axes_0, x = g_cast_fp16)[name = string("g_col_cast_fp16")];
+            bool var_4119_transpose_x_0 = const()[name = string("op_4119_transpose_x_0"), val = bool(false)];
+            bool var_4119_transpose_y_0 = const()[name = string("op_4119_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 16, 64, 1]> var_4119_cast_fp16 = matmul(transpose_x = var_4119_transpose_x_0, transpose_y = var_4119_transpose_y_0, x = layers_0_linear_attn_lower_incl_to_fp16, y = g_col_cast_fp16)[name = string("op_4119_cast_fp16")];
+            tensor<int32, [1]> cum_axes_0 = const()[name = string("cum_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 16, 64]> cum_cast_fp16 = squeeze(axes = cum_axes_0, x = var_4119_cast_fp16)[name = string("cum_cast_fp16")];
+            bool var_4121_transpose_x_0 = const()[name = string("op_4121_transpose_x_0"), val = bool(false)];
+            bool var_4121_transpose_y_0 = const()[name = string("op_4121_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 16, 64, 1]> var_4121_cast_fp16 = matmul(transpose_x = var_4121_transpose_x_0, transpose_y = var_4121_transpose_y_0, x = layers_0_linear_attn_suffix_sum_t_to_fp16, y = g_col_cast_fp16)[name = string("op_4121_cast_fp16")];
+            tensor<int32, [1]> to_end_axes_0 = const()[name = string("to_end_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 16, 64]> to_end_cast_fp16 = squeeze(axes = to_end_axes_0, x = var_4121_cast_fp16)[name = string("to_end_cast_fp16")];
+            bool var_4123_transpose_x_0 = const()[name = string("op_4123_transpose_x_0"), val = bool(false)];
+            bool var_4123_transpose_y_0 = const()[name = string("op_4123_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 16, 4096, 1]> var_4123_cast_fp16 = matmul(transpose_x = var_4123_transpose_x_0, transpose_y = var_4123_transpose_y_0, x = layers_0_linear_attn_pair_sum_t_to_fp16, y = g_col_cast_fp16)[name = string("op_4123_cast_fp16")];
+            tensor<int32, [4]> var_4124 = const()[name = string("op_4124"), val = tensor<int32, [4]>([32, 16, 64, 64])];
+            tensor<fp16, [32, 16, 64, 64]> pair_71_cast_fp16 = reshape(shape = var_4124, x = var_4123_cast_fp16)[name = string("pair_71_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 64]> var_4126_cast_fp16 = exp(x = pair_71_cast_fp16)[name = string("op_4126_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 64]> pair_decay_cast_fp16 = mul(x = var_4126_cast_fp16, y = layers_0_linear_attn_lower_incl_to_fp16)[name = string("pair_decay_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 128]> k_beta_cast_fp16 = mul(x = k_53_cast_fp16, y = beta_cast_fp16)[name = string("k_beta_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 128]> v_beta_cast_fp16 = mul(x = v_27_cast_fp16, y = beta_cast_fp16)[name = string("v_beta_cast_fp16")];
+            bool var_4131_transpose_x_1 = const()[name = string("op_4131_transpose_x_1"), val = bool(false)];
+            bool var_4131_transpose_y_1 = const()[name = string("op_4131_transpose_y_1"), val = bool(true)];
+            tensor<fp16, [32, 16, 64, 64]> var_4131_cast_fp16 = matmul(transpose_x = var_4131_transpose_x_1, transpose_y = var_4131_transpose_y_1, x = k_beta_cast_fp16, y = k_53_cast_fp16)[name = string("op_4131_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 64]> kkt_cast_fp16 = mul(x = var_4131_cast_fp16, y = pair_decay_cast_fp16)[name = string("kkt_cast_fp16")];
+            bool var_4134_transpose_x_1 = const()[name = string("op_4134_transpose_x_1"), val = bool(false)];
+            bool var_4134_transpose_y_1 = const()[name = string("op_4134_transpose_y_1"), val = bool(true)];
+            tensor<fp16, [32, 16, 64, 64]> var_4134_cast_fp16 = matmul(transpose_x = var_4134_transpose_x_1, transpose_y = var_4134_transpose_y_1, x = q_63_cast_fp16, y = k_53_cast_fp16)[name = string("op_4134_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 64]> attn_intra_cast_fp16 = mul(x = var_4134_cast_fp16, y = pair_decay_cast_fp16)[name = string("attn_intra_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 64]> var_4136_cast_fp16 = mul(x = kkt_cast_fp16, y = layers_0_linear_attn_strict_lower_to_fp16)[name = string("op_4136_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 64]> t_21_cast_fp16 = add(x = layers_0_linear_attn_eye_to_fp16, y = var_4136_cast_fp16)[name = string("t_21_cast_fp16")];
+            tensor<int32, [3]> var_4140 = const()[name = string("op_4140"), val = tensor<int32, [3]>([-1, 64, 64])];
+            tensor<fp16, [512, 64, 64]> t_cast_fp16 = reshape(shape = var_4140, x = t_21_cast_fp16)[name = string("t_cast_fp16")];
+            tensor<int32, [5]> var_4145 = const()[name = string("op_4145"), val = tensor<int32, [5]>([512, 32, 2, 32, 2])];
+            tensor<fp16, [512, 32, 2, 32, 2]> var_4146_cast_fp16 = reshape(shape = var_4145, x = t_cast_fp16)[name = string("op_4146_cast_fp16")];
+            tensor<fp16, [512, 32, 2, 32, 2]> var_4151_cast_fp16 = mul(x = var_4146_cast_fp16, y = var_213_to_fp16)[name = string("op_4151_cast_fp16")];
+            tensor<int32, [1]> blocks_61_axes_0 = const()[name = string("blocks_61_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_61_keep_dims_0 = const()[name = string("blocks_61_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 32, 2, 2]> blocks_61_cast_fp16 = reduce_sum(axes = blocks_61_axes_0, keep_dims = blocks_61_keep_dims_0, x = var_4151_cast_fp16)[name = string("blocks_61_cast_fp16")];
+            tensor<int32, [4]> var_4154_begin_0 = const()[name = string("op_4154_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
+            tensor<int32, [4]> var_4154_end_0 = const()[name = string("op_4154_end_0"), val = tensor<int32, [4]>([512, 32, 2, 2])];
+            tensor<bool, [4]> var_4154_end_mask_0 = const()[name = string("op_4154_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 32, 1, 2]> var_4154_cast_fp16 = slice_by_index(begin = var_4154_begin_0, end = var_4154_end_0, end_mask = var_4154_end_mask_0, x = blocks_61_cast_fp16)[name = string("op_4154_cast_fp16")];
+            tensor<int32, [4]> x_157_begin_0 = const()[name = string("x_157_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_157_end_0 = const()[name = string("x_157_end_0"), val = tensor<int32, [4]>([512, 32, 1, 1])];
+            tensor<bool, [4]> x_157_end_mask_0 = const()[name = string("x_157_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 32, 1, 1]> x_157_cast_fp16 = slice_by_index(begin = x_157_begin_0, end = x_157_end_0, end_mask = x_157_end_mask_0, x = var_4154_cast_fp16)[name = string("x_157_cast_fp16")];
+            bool var_4164_transpose_x_0 = const()[name = string("op_4164_transpose_x_0"), val = bool(false)];
+            bool var_4164_transpose_y_0 = const()[name = string("op_4164_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 32, 1, 1]> var_4164_cast_fp16 = matmul(transpose_x = var_4164_transpose_x_0, transpose_y = var_4164_transpose_y_0, x = x_157_cast_fp16, y = var_221_to_fp16)[name = string("op_4164_cast_fp16")];
+            bool var_4165_transpose_x_0 = const()[name = string("op_4165_transpose_x_0"), val = bool(false)];
+            bool var_4165_transpose_y_0 = const()[name = string("op_4165_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 32, 1, 1]> var_4165_cast_fp16 = matmul(transpose_x = var_4165_transpose_x_0, transpose_y = var_4165_transpose_y_0, x = var_221_to_fp16, y = var_4164_cast_fp16)[name = string("op_4165_cast_fp16")];
+            fp16 const_71_promoted_to_fp16 = const()[name = string("const_71_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 32, 1, 1]> lower_61_cast_fp16 = mul(x = var_4165_cast_fp16, y = const_71_promoted_to_fp16)[name = string("lower_61_cast_fp16")];
+            bool bottom_61_interleave_0 = const()[name = string("bottom_61_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 32, 1, 2]> bottom_61_cast_fp16 = concat(axis = var_3986, interleave = bottom_61_interleave_0, values = (lower_61_cast_fp16, var_221_to_fp16))[name = string("bottom_61_cast_fp16")];
+            bool inv_83_interleave_0 = const()[name = string("inv_83_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 32, 2, 2]> inv_83_cast_fp16 = concat(axis = var_3971, interleave = inv_83_interleave_0, values = (top_1_to_fp16, bottom_61_cast_fp16))[name = string("inv_83_cast_fp16")];
+            tensor<int32, [5]> var_4174 = const()[name = string("op_4174"), val = tensor<int32, [5]>([512, 16, 4, 16, 4])];
+            tensor<fp16, [512, 16, 4, 16, 4]> var_4175_cast_fp16 = reshape(shape = var_4174, x = t_cast_fp16)[name = string("op_4175_cast_fp16")];
+            tensor<fp16, [512, 16, 4, 16, 4]> var_4180_cast_fp16 = mul(x = var_4175_cast_fp16, y = var_242_to_fp16)[name = string("op_4180_cast_fp16")];
+            tensor<int32, [1]> blocks_63_axes_0 = const()[name = string("blocks_63_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_63_keep_dims_0 = const()[name = string("blocks_63_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 4, 4]> blocks_63_cast_fp16 = reduce_sum(axes = blocks_63_axes_0, keep_dims = blocks_63_keep_dims_0, x = var_4180_cast_fp16)[name = string("blocks_63_cast_fp16")];
+            tensor<int32, [4]> var_4183_begin_0 = const()[name = string("op_4183_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
+            tensor<int32, [4]> var_4183_end_0 = const()[name = string("op_4183_end_0"), val = tensor<int32, [4]>([512, 16, 4, 4])];
+            tensor<bool, [4]> var_4183_end_mask_0 = const()[name = string("op_4183_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 16, 2, 4]> var_4183_cast_fp16 = slice_by_index(begin = var_4183_begin_0, end = var_4183_end_0, end_mask = var_4183_end_mask_0, x = blocks_63_cast_fp16)[name = string("op_4183_cast_fp16")];
+            tensor<int32, [4]> x_159_begin_0 = const()[name = string("x_159_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_159_end_0 = const()[name = string("x_159_end_0"), val = tensor<int32, [4]>([512, 16, 2, 2])];
+            tensor<bool, [4]> x_159_end_mask_0 = const()[name = string("x_159_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 16, 2, 2]> x_159_cast_fp16 = slice_by_index(begin = x_159_begin_0, end = x_159_end_0, end_mask = x_159_end_mask_0, x = var_4183_cast_fp16)[name = string("x_159_cast_fp16")];
+            tensor<int32, [5]> var_4185 = const()[name = string("op_4185"), val = tensor<int32, [5]>([512, 16, 2, 2, 2])];
+            tensor<fp16, [512, 16, 2, 2, 2]> pair_75_cast_fp16 = reshape(shape = var_4185, x = inv_83_cast_fp16)[name = string("pair_75_cast_fp16")];
+            tensor<int32, [5]> var_4187_begin_0 = const()[name = string("op_4187_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_4187_end_0 = const()[name = string("op_4187_end_0"), val = tensor<int32, [5]>([512, 16, 1, 2, 2])];
+            tensor<bool, [5]> var_4187_end_mask_0 = const()[name = string("op_4187_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_4187_squeeze_mask_0 = const()[name = string("op_4187_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 16, 2, 2]> var_4187_cast_fp16 = slice_by_index(begin = var_4187_begin_0, end = var_4187_end_0, end_mask = var_4187_end_mask_0, squeeze_mask = var_4187_squeeze_mask_0, x = pair_75_cast_fp16)[name = string("op_4187_cast_fp16")];
+            tensor<int32, [5]> var_4190_begin_0 = const()[name = string("op_4190_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_4190_end_0 = const()[name = string("op_4190_end_0"), val = tensor<int32, [5]>([512, 16, 2, 2, 2])];
+            tensor<bool, [5]> var_4190_end_mask_0 = const()[name = string("op_4190_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_4190_squeeze_mask_0 = const()[name = string("op_4190_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 16, 2, 2]> var_4190_cast_fp16 = slice_by_index(begin = var_4190_begin_0, end = var_4190_end_0, end_mask = var_4190_end_mask_0, squeeze_mask = var_4190_squeeze_mask_0, x = pair_75_cast_fp16)[name = string("op_4190_cast_fp16")];
+            bool var_4193_transpose_x_0 = const()[name = string("op_4193_transpose_x_0"), val = bool(false)];
+            bool var_4193_transpose_y_0 = const()[name = string("op_4193_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 2, 2]> var_4193_cast_fp16 = matmul(transpose_x = var_4193_transpose_x_0, transpose_y = var_4193_transpose_y_0, x = x_159_cast_fp16, y = var_4187_cast_fp16)[name = string("op_4193_cast_fp16")];
+            bool var_4194_transpose_x_0 = const()[name = string("op_4194_transpose_x_0"), val = bool(false)];
+            bool var_4194_transpose_y_0 = const()[name = string("op_4194_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 2, 2]> var_4194_cast_fp16 = matmul(transpose_x = var_4194_transpose_x_0, transpose_y = var_4194_transpose_y_0, x = var_4190_cast_fp16, y = var_4193_cast_fp16)[name = string("op_4194_cast_fp16")];
+            fp16 const_72_promoted_to_fp16 = const()[name = string("const_72_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 16, 2, 2]> lower_63_cast_fp16 = mul(x = var_4194_cast_fp16, y = const_72_promoted_to_fp16)[name = string("lower_63_cast_fp16")];
+            bool top_63_interleave_0 = const()[name = string("top_63_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 2, 4]> top_63_cast_fp16 = concat(axis = var_3986, interleave = top_63_interleave_0, values = (var_4187_cast_fp16, var_259_to_fp16))[name = string("top_63_cast_fp16")];
+            bool bottom_63_interleave_0 = const()[name = string("bottom_63_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 2, 4]> bottom_63_cast_fp16 = concat(axis = var_3986, interleave = bottom_63_interleave_0, values = (lower_63_cast_fp16, var_4190_cast_fp16))[name = string("bottom_63_cast_fp16")];
+            bool inv_85_interleave_0 = const()[name = string("inv_85_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 16, 4, 4]> inv_85_cast_fp16 = concat(axis = var_3971, interleave = inv_85_interleave_0, values = (top_63_cast_fp16, bottom_63_cast_fp16))[name = string("inv_85_cast_fp16")];
+            tensor<int32, [5]> var_4203 = const()[name = string("op_4203"), val = tensor<int32, [5]>([512, 8, 8, 8, 8])];
+            tensor<fp16, [512, 8, 8, 8, 8]> var_4204_cast_fp16 = reshape(shape = var_4203, x = t_cast_fp16)[name = string("op_4204_cast_fp16")];
+            tensor<fp16, [8, 1, 8, 1]> var_4208_to_fp16 = const()[name = string("op_4208_to_fp16"), val = tensor<fp16, [8, 1, 8, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(688021696)))];
+            tensor<fp16, [512, 8, 8, 8, 8]> var_4209_cast_fp16 = mul(x = var_4204_cast_fp16, y = var_4208_to_fp16)[name = string("op_4209_cast_fp16")];
+            tensor<int32, [1]> blocks_65_axes_0 = const()[name = string("blocks_65_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_65_keep_dims_0 = const()[name = string("blocks_65_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 8, 8]> blocks_65_cast_fp16 = reduce_sum(axes = blocks_65_axes_0, keep_dims = blocks_65_keep_dims_0, x = var_4209_cast_fp16)[name = string("blocks_65_cast_fp16")];
+            tensor<int32, [4]> var_4212_begin_0 = const()[name = string("op_4212_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
+            tensor<int32, [4]> var_4212_end_0 = const()[name = string("op_4212_end_0"), val = tensor<int32, [4]>([512, 8, 8, 8])];
+            tensor<bool, [4]> var_4212_end_mask_0 = const()[name = string("op_4212_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 8, 4, 8]> var_4212_cast_fp16 = slice_by_index(begin = var_4212_begin_0, end = var_4212_end_0, end_mask = var_4212_end_mask_0, x = blocks_65_cast_fp16)[name = string("op_4212_cast_fp16")];
+            tensor<int32, [4]> x_161_begin_0 = const()[name = string("x_161_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_161_end_0 = const()[name = string("x_161_end_0"), val = tensor<int32, [4]>([512, 8, 4, 4])];
+            tensor<bool, [4]> x_161_end_mask_0 = const()[name = string("x_161_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 8, 4, 4]> x_161_cast_fp16 = slice_by_index(begin = x_161_begin_0, end = x_161_end_0, end_mask = x_161_end_mask_0, x = var_4212_cast_fp16)[name = string("x_161_cast_fp16")];
+            tensor<int32, [5]> var_4214 = const()[name = string("op_4214"), val = tensor<int32, [5]>([512, 8, 2, 4, 4])];
+            tensor<fp16, [512, 8, 2, 4, 4]> pair_77_cast_fp16 = reshape(shape = var_4214, x = inv_85_cast_fp16)[name = string("pair_77_cast_fp16")];
+            tensor<int32, [5]> var_4216_begin_0 = const()[name = string("op_4216_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_4216_end_0 = const()[name = string("op_4216_end_0"), val = tensor<int32, [5]>([512, 8, 1, 4, 4])];
+            tensor<bool, [5]> var_4216_end_mask_0 = const()[name = string("op_4216_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_4216_squeeze_mask_0 = const()[name = string("op_4216_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 8, 4, 4]> var_4216_cast_fp16 = slice_by_index(begin = var_4216_begin_0, end = var_4216_end_0, end_mask = var_4216_end_mask_0, squeeze_mask = var_4216_squeeze_mask_0, x = pair_77_cast_fp16)[name = string("op_4216_cast_fp16")];
+            tensor<int32, [5]> var_4219_begin_0 = const()[name = string("op_4219_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_4219_end_0 = const()[name = string("op_4219_end_0"), val = tensor<int32, [5]>([512, 8, 2, 4, 4])];
+            tensor<bool, [5]> var_4219_end_mask_0 = const()[name = string("op_4219_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_4219_squeeze_mask_0 = const()[name = string("op_4219_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 8, 4, 4]> var_4219_cast_fp16 = slice_by_index(begin = var_4219_begin_0, end = var_4219_end_0, end_mask = var_4219_end_mask_0, squeeze_mask = var_4219_squeeze_mask_0, x = pair_77_cast_fp16)[name = string("op_4219_cast_fp16")];
+            bool var_4222_transpose_x_0 = const()[name = string("op_4222_transpose_x_0"), val = bool(false)];
+            bool var_4222_transpose_y_0 = const()[name = string("op_4222_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 4, 4]> var_4222_cast_fp16 = matmul(transpose_x = var_4222_transpose_x_0, transpose_y = var_4222_transpose_y_0, x = x_161_cast_fp16, y = var_4216_cast_fp16)[name = string("op_4222_cast_fp16")];
+            bool var_4223_transpose_x_0 = const()[name = string("op_4223_transpose_x_0"), val = bool(false)];
+            bool var_4223_transpose_y_0 = const()[name = string("op_4223_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 4, 4]> var_4223_cast_fp16 = matmul(transpose_x = var_4223_transpose_x_0, transpose_y = var_4223_transpose_y_0, x = var_4219_cast_fp16, y = var_4222_cast_fp16)[name = string("op_4223_cast_fp16")];
+            fp16 const_73_promoted_to_fp16 = const()[name = string("const_73_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 8, 4, 4]> lower_65_cast_fp16 = mul(x = var_4223_cast_fp16, y = const_73_promoted_to_fp16)[name = string("lower_65_cast_fp16")];
+            bool top_65_interleave_0 = const()[name = string("top_65_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 4, 8]> top_65_cast_fp16 = concat(axis = var_3986, interleave = top_65_interleave_0, values = (var_4216_cast_fp16, var_288_to_fp16))[name = string("top_65_cast_fp16")];
+            bool bottom_65_interleave_0 = const()[name = string("bottom_65_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 4, 8]> bottom_65_cast_fp16 = concat(axis = var_3986, interleave = bottom_65_interleave_0, values = (lower_65_cast_fp16, var_4219_cast_fp16))[name = string("bottom_65_cast_fp16")];
+            bool inv_87_interleave_0 = const()[name = string("inv_87_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 8, 8, 8]> inv_87_cast_fp16 = concat(axis = var_3971, interleave = inv_87_interleave_0, values = (top_65_cast_fp16, bottom_65_cast_fp16))[name = string("inv_87_cast_fp16")];
+            tensor<int32, [5]> var_4232 = const()[name = string("op_4232"), val = tensor<int32, [5]>([512, 4, 16, 4, 16])];
+            tensor<fp16, [512, 4, 16, 4, 16]> var_4233_cast_fp16 = reshape(shape = var_4232, x = t_cast_fp16)[name = string("op_4233_cast_fp16")];
+            tensor<fp16, [4, 1, 4, 1]> var_4237_to_fp16 = const()[name = string("op_4237_to_fp16"), val = tensor<fp16, [4, 1, 4, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(688021888)))];
+            tensor<fp16, [512, 4, 16, 4, 16]> var_4238_cast_fp16 = mul(x = var_4233_cast_fp16, y = var_4237_to_fp16)[name = string("op_4238_cast_fp16")];
+            tensor<int32, [1]> blocks_67_axes_0 = const()[name = string("blocks_67_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_67_keep_dims_0 = const()[name = string("blocks_67_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 16, 16]> blocks_67_cast_fp16 = reduce_sum(axes = blocks_67_axes_0, keep_dims = blocks_67_keep_dims_0, x = var_4238_cast_fp16)[name = string("blocks_67_cast_fp16")];
+            tensor<int32, [4]> var_4241_begin_0 = const()[name = string("op_4241_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
+            tensor<int32, [4]> var_4241_end_0 = const()[name = string("op_4241_end_0"), val = tensor<int32, [4]>([512, 4, 16, 16])];
+            tensor<bool, [4]> var_4241_end_mask_0 = const()[name = string("op_4241_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 4, 8, 16]> var_4241_cast_fp16 = slice_by_index(begin = var_4241_begin_0, end = var_4241_end_0, end_mask = var_4241_end_mask_0, x = blocks_67_cast_fp16)[name = string("op_4241_cast_fp16")];
+            tensor<int32, [4]> x_163_begin_0 = const()[name = string("x_163_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_163_end_0 = const()[name = string("x_163_end_0"), val = tensor<int32, [4]>([512, 4, 8, 8])];
+            tensor<bool, [4]> x_163_end_mask_0 = const()[name = string("x_163_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 4, 8, 8]> x_163_cast_fp16 = slice_by_index(begin = x_163_begin_0, end = x_163_end_0, end_mask = x_163_end_mask_0, x = var_4241_cast_fp16)[name = string("x_163_cast_fp16")];
+            tensor<int32, [5]> var_4243 = const()[name = string("op_4243"), val = tensor<int32, [5]>([512, 4, 2, 8, 8])];
+            tensor<fp16, [512, 4, 2, 8, 8]> pair_79_cast_fp16 = reshape(shape = var_4243, x = inv_87_cast_fp16)[name = string("pair_79_cast_fp16")];
+            tensor<int32, [5]> var_4245_begin_0 = const()[name = string("op_4245_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_4245_end_0 = const()[name = string("op_4245_end_0"), val = tensor<int32, [5]>([512, 4, 1, 8, 8])];
+            tensor<bool, [5]> var_4245_end_mask_0 = const()[name = string("op_4245_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_4245_squeeze_mask_0 = const()[name = string("op_4245_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 4, 8, 8]> var_4245_cast_fp16 = slice_by_index(begin = var_4245_begin_0, end = var_4245_end_0, end_mask = var_4245_end_mask_0, squeeze_mask = var_4245_squeeze_mask_0, x = pair_79_cast_fp16)[name = string("op_4245_cast_fp16")];
+            tensor<int32, [5]> var_4248_begin_0 = const()[name = string("op_4248_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_4248_end_0 = const()[name = string("op_4248_end_0"), val = tensor<int32, [5]>([512, 4, 2, 8, 8])];
+            tensor<bool, [5]> var_4248_end_mask_0 = const()[name = string("op_4248_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_4248_squeeze_mask_0 = const()[name = string("op_4248_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 4, 8, 8]> var_4248_cast_fp16 = slice_by_index(begin = var_4248_begin_0, end = var_4248_end_0, end_mask = var_4248_end_mask_0, squeeze_mask = var_4248_squeeze_mask_0, x = pair_79_cast_fp16)[name = string("op_4248_cast_fp16")];
+            bool var_4251_transpose_x_0 = const()[name = string("op_4251_transpose_x_0"), val = bool(false)];
+            bool var_4251_transpose_y_0 = const()[name = string("op_4251_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 8, 8]> var_4251_cast_fp16 = matmul(transpose_x = var_4251_transpose_x_0, transpose_y = var_4251_transpose_y_0, x = x_163_cast_fp16, y = var_4245_cast_fp16)[name = string("op_4251_cast_fp16")];
+            bool var_4252_transpose_x_0 = const()[name = string("op_4252_transpose_x_0"), val = bool(false)];
+            bool var_4252_transpose_y_0 = const()[name = string("op_4252_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 8, 8]> var_4252_cast_fp16 = matmul(transpose_x = var_4252_transpose_x_0, transpose_y = var_4252_transpose_y_0, x = var_4248_cast_fp16, y = var_4251_cast_fp16)[name = string("op_4252_cast_fp16")];
+            fp16 const_74_promoted_to_fp16 = const()[name = string("const_74_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 4, 8, 8]> lower_67_cast_fp16 = mul(x = var_4252_cast_fp16, y = const_74_promoted_to_fp16)[name = string("lower_67_cast_fp16")];
+            bool top_67_interleave_0 = const()[name = string("top_67_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 8, 16]> top_67_cast_fp16 = concat(axis = var_3986, interleave = top_67_interleave_0, values = (var_4245_cast_fp16, var_317_to_fp16))[name = string("top_67_cast_fp16")];
+            bool bottom_67_interleave_0 = const()[name = string("bottom_67_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 8, 16]> bottom_67_cast_fp16 = concat(axis = var_3986, interleave = bottom_67_interleave_0, values = (lower_67_cast_fp16, var_4248_cast_fp16))[name = string("bottom_67_cast_fp16")];
+            bool inv_89_interleave_0 = const()[name = string("inv_89_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 4, 16, 16]> inv_89_cast_fp16 = concat(axis = var_3971, interleave = inv_89_interleave_0, values = (top_67_cast_fp16, bottom_67_cast_fp16))[name = string("inv_89_cast_fp16")];
+            tensor<int32, [5]> var_4261 = const()[name = string("op_4261"), val = tensor<int32, [5]>([512, 2, 32, 2, 32])];
+            tensor<fp16, [512, 2, 32, 2, 32]> var_4262_cast_fp16 = reshape(shape = var_4261, x = t_cast_fp16)[name = string("op_4262_cast_fp16")];
+            tensor<fp16, [2, 1, 2, 1]> var_4266_to_fp16 = const()[name = string("op_4266_to_fp16"), val = tensor<fp16, [2, 1, 2, 1]>([[[[0x1p+0], [0x0p+0]]], [[[0x0p+0], [0x1p+0]]]])];
+            tensor<fp16, [512, 2, 32, 2, 32]> var_4267_cast_fp16 = mul(x = var_4262_cast_fp16, y = var_4266_to_fp16)[name = string("op_4267_cast_fp16")];
+            tensor<int32, [1]> blocks_69_axes_0 = const()[name = string("blocks_69_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_69_keep_dims_0 = const()[name = string("blocks_69_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 32, 32]> blocks_69_cast_fp16 = reduce_sum(axes = blocks_69_axes_0, keep_dims = blocks_69_keep_dims_0, x = var_4267_cast_fp16)[name = string("blocks_69_cast_fp16")];
+            tensor<int32, [4]> var_4270_begin_0 = const()[name = string("op_4270_begin_0"), val = tensor<int32, [4]>([0, 0, 16, 0])];
+            tensor<int32, [4]> var_4270_end_0 = const()[name = string("op_4270_end_0"), val = tensor<int32, [4]>([512, 2, 32, 32])];
+            tensor<bool, [4]> var_4270_end_mask_0 = const()[name = string("op_4270_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 2, 16, 32]> var_4270_cast_fp16 = slice_by_index(begin = var_4270_begin_0, end = var_4270_end_0, end_mask = var_4270_end_mask_0, x = blocks_69_cast_fp16)[name = string("op_4270_cast_fp16")];
+            tensor<int32, [4]> x_165_begin_0 = const()[name = string("x_165_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_165_end_0 = const()[name = string("x_165_end_0"), val = tensor<int32, [4]>([512, 2, 16, 16])];
+            tensor<bool, [4]> x_165_end_mask_0 = const()[name = string("x_165_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 2, 16, 16]> x_165_cast_fp16 = slice_by_index(begin = x_165_begin_0, end = x_165_end_0, end_mask = x_165_end_mask_0, x = var_4270_cast_fp16)[name = string("x_165_cast_fp16")];
+            tensor<int32, [5]> var_4272 = const()[name = string("op_4272"), val = tensor<int32, [5]>([512, 2, 2, 16, 16])];
+            tensor<fp16, [512, 2, 2, 16, 16]> pair_81_cast_fp16 = reshape(shape = var_4272, x = inv_89_cast_fp16)[name = string("pair_81_cast_fp16")];
+            tensor<int32, [5]> var_4274_begin_0 = const()[name = string("op_4274_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_4274_end_0 = const()[name = string("op_4274_end_0"), val = tensor<int32, [5]>([512, 2, 1, 16, 16])];
+            tensor<bool, [5]> var_4274_end_mask_0 = const()[name = string("op_4274_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_4274_squeeze_mask_0 = const()[name = string("op_4274_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 2, 16, 16]> var_4274_cast_fp16 = slice_by_index(begin = var_4274_begin_0, end = var_4274_end_0, end_mask = var_4274_end_mask_0, squeeze_mask = var_4274_squeeze_mask_0, x = pair_81_cast_fp16)[name = string("op_4274_cast_fp16")];
+            tensor<int32, [5]> var_4277_begin_0 = const()[name = string("op_4277_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_4277_end_0 = const()[name = string("op_4277_end_0"), val = tensor<int32, [5]>([512, 2, 2, 16, 16])];
+            tensor<bool, [5]> var_4277_end_mask_0 = const()[name = string("op_4277_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_4277_squeeze_mask_0 = const()[name = string("op_4277_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 2, 16, 16]> var_4277_cast_fp16 = slice_by_index(begin = var_4277_begin_0, end = var_4277_end_0, end_mask = var_4277_end_mask_0, squeeze_mask = var_4277_squeeze_mask_0, x = pair_81_cast_fp16)[name = string("op_4277_cast_fp16")];
+            bool var_4280_transpose_x_0 = const()[name = string("op_4280_transpose_x_0"), val = bool(false)];
+            bool var_4280_transpose_y_0 = const()[name = string("op_4280_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 16, 16]> var_4280_cast_fp16 = matmul(transpose_x = var_4280_transpose_x_0, transpose_y = var_4280_transpose_y_0, x = x_165_cast_fp16, y = var_4274_cast_fp16)[name = string("op_4280_cast_fp16")];
+            bool var_4281_transpose_x_0 = const()[name = string("op_4281_transpose_x_0"), val = bool(false)];
+            bool var_4281_transpose_y_0 = const()[name = string("op_4281_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 16, 16]> var_4281_cast_fp16 = matmul(transpose_x = var_4281_transpose_x_0, transpose_y = var_4281_transpose_y_0, x = var_4277_cast_fp16, y = var_4280_cast_fp16)[name = string("op_4281_cast_fp16")];
+            fp16 const_75_promoted_to_fp16 = const()[name = string("const_75_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 2, 16, 16]> lower_69_cast_fp16 = mul(x = var_4281_cast_fp16, y = const_75_promoted_to_fp16)[name = string("lower_69_cast_fp16")];
+            bool top_69_interleave_0 = const()[name = string("top_69_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 16, 32]> top_69_cast_fp16 = concat(axis = var_3986, interleave = top_69_interleave_0, values = (var_4274_cast_fp16, var_346_to_fp16))[name = string("top_69_cast_fp16")];
+            bool bottom_69_interleave_0 = const()[name = string("bottom_69_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 16, 32]> bottom_69_cast_fp16 = concat(axis = var_3986, interleave = bottom_69_interleave_0, values = (lower_69_cast_fp16, var_4277_cast_fp16))[name = string("bottom_69_cast_fp16")];
+            bool inv_91_interleave_0 = const()[name = string("inv_91_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 2, 32, 32]> inv_91_cast_fp16 = concat(axis = var_3971, interleave = inv_91_interleave_0, values = (top_69_cast_fp16, bottom_69_cast_fp16))[name = string("inv_91_cast_fp16")];
+            tensor<int32, [5]> var_4290 = const()[name = string("op_4290"), val = tensor<int32, [5]>([512, 1, 64, 1, 64])];
+            tensor<fp16, [512, 1, 64, 1, 64]> var_4291_cast_fp16 = reshape(shape = var_4290, x = t_cast_fp16)[name = string("op_4291_cast_fp16")];
+            tensor<int32, [1]> blocks_axes_0 = const()[name = string("blocks_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_keep_dims_0 = const()[name = string("blocks_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 64, 64]> blocks_cast_fp16 = reduce_sum(axes = blocks_axes_0, keep_dims = blocks_keep_dims_0, x = var_4291_cast_fp16)[name = string("blocks_cast_fp16")];
+            tensor<int32, [4]> var_4299_begin_0 = const()[name = string("op_4299_begin_0"), val = tensor<int32, [4]>([0, 0, 32, 0])];
+            tensor<int32, [4]> var_4299_end_0 = const()[name = string("op_4299_end_0"), val = tensor<int32, [4]>([512, 1, 64, 64])];
+            tensor<bool, [4]> var_4299_end_mask_0 = const()[name = string("op_4299_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [512, 1, 32, 64]> var_4299_cast_fp16 = slice_by_index(begin = var_4299_begin_0, end = var_4299_end_0, end_mask = var_4299_end_mask_0, x = blocks_cast_fp16)[name = string("op_4299_cast_fp16")];
+            tensor<int32, [4]> x_167_begin_0 = const()[name = string("x_167_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_167_end_0 = const()[name = string("x_167_end_0"), val = tensor<int32, [4]>([512, 1, 32, 32])];
+            tensor<bool, [4]> x_167_end_mask_0 = const()[name = string("x_167_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [512, 1, 32, 32]> x_167_cast_fp16 = slice_by_index(begin = x_167_begin_0, end = x_167_end_0, end_mask = x_167_end_mask_0, x = var_4299_cast_fp16)[name = string("x_167_cast_fp16")];
+            tensor<int32, [5]> var_4301 = const()[name = string("op_4301"), val = tensor<int32, [5]>([512, 1, 2, 32, 32])];
+            tensor<fp16, [512, 1, 2, 32, 32]> pair_cast_fp16 = reshape(shape = var_4301, x = inv_91_cast_fp16)[name = string("pair_cast_fp16")];
+            tensor<int32, [5]> var_4303_begin_0 = const()[name = string("op_4303_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_4303_end_0 = const()[name = string("op_4303_end_0"), val = tensor<int32, [5]>([512, 1, 1, 32, 32])];
+            tensor<bool, [5]> var_4303_end_mask_0 = const()[name = string("op_4303_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_4303_squeeze_mask_0 = const()[name = string("op_4303_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 1, 32, 32]> var_4303_cast_fp16 = slice_by_index(begin = var_4303_begin_0, end = var_4303_end_0, end_mask = var_4303_end_mask_0, squeeze_mask = var_4303_squeeze_mask_0, x = pair_cast_fp16)[name = string("op_4303_cast_fp16")];
+            tensor<int32, [5]> var_4306_begin_0 = const()[name = string("op_4306_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_4306_end_0 = const()[name = string("op_4306_end_0"), val = tensor<int32, [5]>([512, 1, 2, 32, 32])];
+            tensor<bool, [5]> var_4306_end_mask_0 = const()[name = string("op_4306_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_4306_squeeze_mask_0 = const()[name = string("op_4306_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [512, 1, 32, 32]> var_4306_cast_fp16 = slice_by_index(begin = var_4306_begin_0, end = var_4306_end_0, end_mask = var_4306_end_mask_0, squeeze_mask = var_4306_squeeze_mask_0, x = pair_cast_fp16)[name = string("op_4306_cast_fp16")];
+            bool var_4309_transpose_x_0 = const()[name = string("op_4309_transpose_x_0"), val = bool(false)];
+            bool var_4309_transpose_y_0 = const()[name = string("op_4309_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 32, 32]> var_4309_cast_fp16 = matmul(transpose_x = var_4309_transpose_x_0, transpose_y = var_4309_transpose_y_0, x = x_167_cast_fp16, y = var_4303_cast_fp16)[name = string("op_4309_cast_fp16")];
+            bool var_4310_transpose_x_0 = const()[name = string("op_4310_transpose_x_0"), val = bool(false)];
+            bool var_4310_transpose_y_0 = const()[name = string("op_4310_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 32, 32]> var_4310_cast_fp16 = matmul(transpose_x = var_4310_transpose_x_0, transpose_y = var_4310_transpose_y_0, x = var_4306_cast_fp16, y = var_4309_cast_fp16)[name = string("op_4310_cast_fp16")];
+            fp16 const_76_promoted_to_fp16 = const()[name = string("const_76_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [512, 1, 32, 32]> lower_cast_fp16 = mul(x = var_4310_cast_fp16, y = const_76_promoted_to_fp16)[name = string("lower_cast_fp16")];
+            bool top_interleave_0 = const()[name = string("top_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 32, 64]> top_cast_fp16 = concat(axis = var_3986, interleave = top_interleave_0, values = (var_4303_cast_fp16, var_375_to_fp16))[name = string("top_cast_fp16")];
+            bool bottom_interleave_0 = const()[name = string("bottom_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 32, 64]> bottom_cast_fp16 = concat(axis = var_3986, interleave = bottom_interleave_0, values = (lower_cast_fp16, var_4306_cast_fp16))[name = string("bottom_cast_fp16")];
+            bool inv_93_interleave_0 = const()[name = string("inv_93_interleave_0"), val = bool(false)];
+            tensor<fp16, [512, 1, 64, 64]> inv_93_cast_fp16 = concat(axis = var_3971, interleave = inv_93_interleave_0, values = (top_cast_fp16, bottom_cast_fp16))[name = string("inv_93_cast_fp16")];
+            tensor<int32, [4]> var_4319 = const()[name = string("op_4319"), val = tensor<int32, [4]>([32, 16, 64, 64])];
+            tensor<fp16, [32, 16, 64, 64]> inv_cast_fp16 = reshape(shape = var_4319, x = inv_93_cast_fp16)[name = string("inv_cast_fp16")];
+            bool new_v_transpose_x_0 = const()[name = string("new_v_transpose_x_0"), val = bool(false)];
+            bool new_v_transpose_y_0 = const()[name = string("new_v_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 16, 64, 128]> new_v_cast_fp16 = matmul(transpose_x = new_v_transpose_x_0, transpose_y = new_v_transpose_y_0, x = inv_cast_fp16, y = v_beta_cast_fp16)[name = string("new_v_cast_fp16")];
+            tensor<fp16, [32, 16, 64]> var_4322_cast_fp16 = exp(x = cum_cast_fp16)[name = string("op_4322_cast_fp16")];
+            tensor<int32, [1]> var_4323_axes_0 = const()[name = string("op_4323_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 16, 64, 1]> var_4323_cast_fp16 = expand_dims(axes = var_4323_axes_0, x = var_4322_cast_fp16)[name = string("op_4323_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 128]> var_4324_cast_fp16 = mul(x = k_beta_cast_fp16, y = var_4323_cast_fp16)[name = string("op_4324_cast_fp16")];
+            bool k_cumdecay_transpose_x_0 = const()[name = string("k_cumdecay_transpose_x_0"), val = bool(false)];
+            bool k_cumdecay_transpose_y_0 = const()[name = string("k_cumdecay_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 16, 64, 128]> k_cumdecay_cast_fp16 = matmul(transpose_x = k_cumdecay_transpose_x_0, transpose_y = k_cumdecay_transpose_y_0, x = inv_cast_fp16, y = var_4324_cast_fp16)[name = string("k_cumdecay_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 128]> q_dec_cast_fp16 = mul(x = q_63_cast_fp16, y = var_4323_cast_fp16)[name = string("q_dec_cast_fp16")];
+            tensor<fp16, [32, 16, 64]> var_4329_cast_fp16 = exp(x = to_end_cast_fp16)[name = string("op_4329_cast_fp16")];
+            tensor<int32, [1]> var_4330_axes_0 = const()[name = string("op_4330_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 16, 64, 1]> var_4330_cast_fp16 = expand_dims(axes = var_4330_axes_0, x = var_4329_cast_fp16)[name = string("op_4330_cast_fp16")];
+            tensor<fp16, [32, 16, 64, 128]> k_dec_cast_fp16 = mul(x = k_53_cast_fp16, y = var_4330_cast_fp16)[name = string("k_dec_cast_fp16")];
+            tensor<int32, [3]> var_4332_begin_0 = const()[name = string("op_4332_begin_0"), val = tensor<int32, [3]>([0, 0, -1])];
+            tensor<int32, [3]> var_4332_end_0 = const()[name = string("op_4332_end_0"), val = tensor<int32, [3]>([32, 16, 64])];
+            tensor<bool, [3]> var_4332_end_mask_0 = const()[name = string("op_4332_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
+            tensor<fp16, [32, 16, 1]> var_4332_cast_fp16 = slice_by_index(begin = var_4332_begin_0, end = var_4332_end_0, end_mask = var_4332_end_mask_0, x = cum_cast_fp16)[name = string("op_4332_cast_fp16")];
+            tensor<fp16, [32, 16, 1]> var_4333_cast_fp16 = exp(x = var_4332_cast_fp16)[name = string("op_4333_cast_fp16")];
+            tensor<int32, [1]> chunk_decay_axes_0 = const()[name = string("chunk_decay_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 16, 1, 1]> chunk_decay_cast_fp16 = expand_dims(axes = chunk_decay_axes_0, x = var_4333_cast_fp16)[name = string("chunk_decay_cast_fp16")];
+            tensor<int32, [4]> var_4338_begin_0 = const()[name = string("op_4338_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_4338_end_0 = const()[name = string("op_4338_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_4338_end_mask_0 = const()[name = string("op_4338_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4338_squeeze_mask_0 = const()[name = string("op_4338_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4338_cast_fp16 = slice_by_index(begin = var_4338_begin_0, end = var_4338_end_0, end_mask = var_4338_end_mask_0, squeeze_mask = var_4338_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_4338_cast_fp16")];
+            tensor<int32, [4]> var_4340_begin_0 = const()[name = string("op_4340_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_4340_end_0 = const()[name = string("op_4340_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_4340_end_mask_0 = const()[name = string("op_4340_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4340_squeeze_mask_0 = const()[name = string("op_4340_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4340_cast_fp16 = slice_by_index(begin = var_4340_begin_0, end = var_4340_end_0, end_mask = var_4340_end_mask_0, squeeze_mask = var_4340_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_4340_cast_fp16")];
+            bool var_4341_transpose_x_0 = const()[name = string("op_4341_transpose_x_0"), val = bool(false)];
+            bool var_4341_transpose_y_0 = const()[name = string("op_4341_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4341_cast_fp16 = matmul(transpose_x = var_4341_transpose_x_0, transpose_y = var_4341_transpose_y_0, x = var_4340_cast_fp16, y = state_1_to_fp16)[name = string("op_4341_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_161_cast_fp16 = sub(x = var_4338_cast_fp16, y = var_4341_cast_fp16)[name = string("v_new_161_cast_fp16")];
+            tensor<int32, [4]> var_4344_begin_0 = const()[name = string("op_4344_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_4344_end_0 = const()[name = string("op_4344_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_4344_end_mask_0 = const()[name = string("op_4344_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4344_squeeze_mask_0 = const()[name = string("op_4344_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4344_cast_fp16 = slice_by_index(begin = var_4344_begin_0, end = var_4344_end_0, end_mask = var_4344_end_mask_0, squeeze_mask = var_4344_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_4344_cast_fp16")];
+            bool var_4345_transpose_x_0 = const()[name = string("op_4345_transpose_x_0"), val = bool(false)];
+            bool var_4345_transpose_y_0 = const()[name = string("op_4345_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4345_cast_fp16 = matmul(transpose_x = var_4345_transpose_x_0, transpose_y = var_4345_transpose_y_0, x = var_4344_cast_fp16, y = state_1_to_fp16)[name = string("op_4345_cast_fp16")];
+            tensor<int32, [4]> var_4347_begin_0 = const()[name = string("op_4347_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_4347_end_0 = const()[name = string("op_4347_end_0"), val = tensor<int32, [4]>([32, 1, 64, 64])];
+            tensor<bool, [4]> var_4347_end_mask_0 = const()[name = string("op_4347_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4347_squeeze_mask_0 = const()[name = string("op_4347_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4347_cast_fp16 = slice_by_index(begin = var_4347_begin_0, end = var_4347_end_0, end_mask = var_4347_end_mask_0, squeeze_mask = var_4347_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_4347_cast_fp16")];
+            bool var_4348_transpose_x_0 = const()[name = string("op_4348_transpose_x_0"), val = bool(false)];
+            bool var_4348_transpose_y_0 = const()[name = string("op_4348_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4348_cast_fp16 = matmul(transpose_x = var_4348_transpose_x_0, transpose_y = var_4348_transpose_y_0, x = var_4347_cast_fp16, y = v_new_161_cast_fp16)[name = string("op_4348_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4349_cast_fp16 = add(x = var_4345_cast_fp16, y = var_4348_cast_fp16)[name = string("op_4349_cast_fp16")];
+            tensor<int32, [4]> var_4351_begin_0 = const()[name = string("op_4351_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_4351_end_0 = const()[name = string("op_4351_end_0"), val = tensor<int32, [4]>([32, 1, 1, 1])];
+            tensor<bool, [4]> var_4351_end_mask_0 = const()[name = string("op_4351_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4351_squeeze_mask_0 = const()[name = string("op_4351_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4351_cast_fp16 = slice_by_index(begin = var_4351_begin_0, end = var_4351_end_0, end_mask = var_4351_end_mask_0, squeeze_mask = var_4351_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_4351_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4352_cast_fp16 = mul(x = state_1_to_fp16, y = var_4351_cast_fp16)[name = string("op_4352_cast_fp16")];
+            tensor<int32, [4]> var_4354_begin_0 = const()[name = string("op_4354_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_4354_end_0 = const()[name = string("op_4354_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_4354_end_mask_0 = const()[name = string("op_4354_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4354_squeeze_mask_0 = const()[name = string("op_4354_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4354_cast_fp16 = slice_by_index(begin = var_4354_begin_0, end = var_4354_end_0, end_mask = var_4354_end_mask_0, squeeze_mask = var_4354_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_4354_cast_fp16")];
+            bool var_4356_transpose_x_1 = const()[name = string("op_4356_transpose_x_1"), val = bool(true)];
+            bool var_4356_transpose_y_1 = const()[name = string("op_4356_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4356_cast_fp16 = matmul(transpose_x = var_4356_transpose_x_1, transpose_y = var_4356_transpose_y_1, x = var_4354_cast_fp16, y = v_new_161_cast_fp16)[name = string("op_4356_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_163_cast_fp16 = add(x = var_4352_cast_fp16, y = var_4356_cast_fp16)[name = string("state_163_cast_fp16")];
+            tensor<int32, [4]> var_4359_begin_0 = const()[name = string("op_4359_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_4359_end_0 = const()[name = string("op_4359_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_4359_end_mask_0 = const()[name = string("op_4359_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4359_squeeze_mask_0 = const()[name = string("op_4359_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4359_cast_fp16 = slice_by_index(begin = var_4359_begin_0, end = var_4359_end_0, end_mask = var_4359_end_mask_0, squeeze_mask = var_4359_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_4359_cast_fp16")];
+            tensor<int32, [4]> var_4361_begin_0 = const()[name = string("op_4361_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_4361_end_0 = const()[name = string("op_4361_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_4361_end_mask_0 = const()[name = string("op_4361_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4361_squeeze_mask_0 = const()[name = string("op_4361_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4361_cast_fp16 = slice_by_index(begin = var_4361_begin_0, end = var_4361_end_0, end_mask = var_4361_end_mask_0, squeeze_mask = var_4361_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_4361_cast_fp16")];
+            bool var_4362_transpose_x_0 = const()[name = string("op_4362_transpose_x_0"), val = bool(false)];
+            bool var_4362_transpose_y_0 = const()[name = string("op_4362_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4362_cast_fp16 = matmul(transpose_x = var_4362_transpose_x_0, transpose_y = var_4362_transpose_y_0, x = var_4361_cast_fp16, y = state_163_cast_fp16)[name = string("op_4362_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_163_cast_fp16 = sub(x = var_4359_cast_fp16, y = var_4362_cast_fp16)[name = string("v_new_163_cast_fp16")];
+            tensor<int32, [4]> var_4365_begin_0 = const()[name = string("op_4365_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_4365_end_0 = const()[name = string("op_4365_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_4365_end_mask_0 = const()[name = string("op_4365_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4365_squeeze_mask_0 = const()[name = string("op_4365_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4365_cast_fp16 = slice_by_index(begin = var_4365_begin_0, end = var_4365_end_0, end_mask = var_4365_end_mask_0, squeeze_mask = var_4365_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_4365_cast_fp16")];
+            bool var_4366_transpose_x_0 = const()[name = string("op_4366_transpose_x_0"), val = bool(false)];
+            bool var_4366_transpose_y_0 = const()[name = string("op_4366_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4366_cast_fp16 = matmul(transpose_x = var_4366_transpose_x_0, transpose_y = var_4366_transpose_y_0, x = var_4365_cast_fp16, y = state_163_cast_fp16)[name = string("op_4366_cast_fp16")];
+            tensor<int32, [4]> var_4368_begin_0 = const()[name = string("op_4368_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_4368_end_0 = const()[name = string("op_4368_end_0"), val = tensor<int32, [4]>([32, 2, 64, 64])];
+            tensor<bool, [4]> var_4368_end_mask_0 = const()[name = string("op_4368_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4368_squeeze_mask_0 = const()[name = string("op_4368_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4368_cast_fp16 = slice_by_index(begin = var_4368_begin_0, end = var_4368_end_0, end_mask = var_4368_end_mask_0, squeeze_mask = var_4368_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_4368_cast_fp16")];
+            bool var_4369_transpose_x_0 = const()[name = string("op_4369_transpose_x_0"), val = bool(false)];
+            bool var_4369_transpose_y_0 = const()[name = string("op_4369_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4369_cast_fp16 = matmul(transpose_x = var_4369_transpose_x_0, transpose_y = var_4369_transpose_y_0, x = var_4368_cast_fp16, y = v_new_163_cast_fp16)[name = string("op_4369_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4370_cast_fp16 = add(x = var_4366_cast_fp16, y = var_4369_cast_fp16)[name = string("op_4370_cast_fp16")];
+            tensor<int32, [4]> var_4372_begin_0 = const()[name = string("op_4372_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_4372_end_0 = const()[name = string("op_4372_end_0"), val = tensor<int32, [4]>([32, 2, 1, 1])];
+            tensor<bool, [4]> var_4372_end_mask_0 = const()[name = string("op_4372_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4372_squeeze_mask_0 = const()[name = string("op_4372_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4372_cast_fp16 = slice_by_index(begin = var_4372_begin_0, end = var_4372_end_0, end_mask = var_4372_end_mask_0, squeeze_mask = var_4372_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_4372_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4373_cast_fp16 = mul(x = state_163_cast_fp16, y = var_4372_cast_fp16)[name = string("op_4373_cast_fp16")];
+            tensor<int32, [4]> var_4375_begin_0 = const()[name = string("op_4375_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_4375_end_0 = const()[name = string("op_4375_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_4375_end_mask_0 = const()[name = string("op_4375_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4375_squeeze_mask_0 = const()[name = string("op_4375_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4375_cast_fp16 = slice_by_index(begin = var_4375_begin_0, end = var_4375_end_0, end_mask = var_4375_end_mask_0, squeeze_mask = var_4375_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_4375_cast_fp16")];
+            bool var_4377_transpose_x_1 = const()[name = string("op_4377_transpose_x_1"), val = bool(true)];
+            bool var_4377_transpose_y_1 = const()[name = string("op_4377_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4377_cast_fp16 = matmul(transpose_x = var_4377_transpose_x_1, transpose_y = var_4377_transpose_y_1, x = var_4375_cast_fp16, y = v_new_163_cast_fp16)[name = string("op_4377_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_165_cast_fp16 = add(x = var_4373_cast_fp16, y = var_4377_cast_fp16)[name = string("state_165_cast_fp16")];
+            tensor<int32, [4]> var_4380_begin_0 = const()[name = string("op_4380_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_4380_end_0 = const()[name = string("op_4380_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_4380_end_mask_0 = const()[name = string("op_4380_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4380_squeeze_mask_0 = const()[name = string("op_4380_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4380_cast_fp16 = slice_by_index(begin = var_4380_begin_0, end = var_4380_end_0, end_mask = var_4380_end_mask_0, squeeze_mask = var_4380_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_4380_cast_fp16")];
+            tensor<int32, [4]> var_4382_begin_0 = const()[name = string("op_4382_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_4382_end_0 = const()[name = string("op_4382_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_4382_end_mask_0 = const()[name = string("op_4382_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4382_squeeze_mask_0 = const()[name = string("op_4382_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4382_cast_fp16 = slice_by_index(begin = var_4382_begin_0, end = var_4382_end_0, end_mask = var_4382_end_mask_0, squeeze_mask = var_4382_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_4382_cast_fp16")];
+            bool var_4383_transpose_x_0 = const()[name = string("op_4383_transpose_x_0"), val = bool(false)];
+            bool var_4383_transpose_y_0 = const()[name = string("op_4383_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4383_cast_fp16 = matmul(transpose_x = var_4383_transpose_x_0, transpose_y = var_4383_transpose_y_0, x = var_4382_cast_fp16, y = state_165_cast_fp16)[name = string("op_4383_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_165_cast_fp16 = sub(x = var_4380_cast_fp16, y = var_4383_cast_fp16)[name = string("v_new_165_cast_fp16")];
+            tensor<int32, [4]> var_4386_begin_0 = const()[name = string("op_4386_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_4386_end_0 = const()[name = string("op_4386_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_4386_end_mask_0 = const()[name = string("op_4386_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4386_squeeze_mask_0 = const()[name = string("op_4386_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4386_cast_fp16 = slice_by_index(begin = var_4386_begin_0, end = var_4386_end_0, end_mask = var_4386_end_mask_0, squeeze_mask = var_4386_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_4386_cast_fp16")];
+            bool var_4387_transpose_x_0 = const()[name = string("op_4387_transpose_x_0"), val = bool(false)];
+            bool var_4387_transpose_y_0 = const()[name = string("op_4387_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4387_cast_fp16 = matmul(transpose_x = var_4387_transpose_x_0, transpose_y = var_4387_transpose_y_0, x = var_4386_cast_fp16, y = state_165_cast_fp16)[name = string("op_4387_cast_fp16")];
+            tensor<int32, [4]> var_4389_begin_0 = const()[name = string("op_4389_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_4389_end_0 = const()[name = string("op_4389_end_0"), val = tensor<int32, [4]>([32, 3, 64, 64])];
+            tensor<bool, [4]> var_4389_end_mask_0 = const()[name = string("op_4389_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4389_squeeze_mask_0 = const()[name = string("op_4389_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4389_cast_fp16 = slice_by_index(begin = var_4389_begin_0, end = var_4389_end_0, end_mask = var_4389_end_mask_0, squeeze_mask = var_4389_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_4389_cast_fp16")];
+            bool var_4390_transpose_x_0 = const()[name = string("op_4390_transpose_x_0"), val = bool(false)];
+            bool var_4390_transpose_y_0 = const()[name = string("op_4390_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4390_cast_fp16 = matmul(transpose_x = var_4390_transpose_x_0, transpose_y = var_4390_transpose_y_0, x = var_4389_cast_fp16, y = v_new_165_cast_fp16)[name = string("op_4390_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4391_cast_fp16 = add(x = var_4387_cast_fp16, y = var_4390_cast_fp16)[name = string("op_4391_cast_fp16")];
+            tensor<int32, [4]> var_4393_begin_0 = const()[name = string("op_4393_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_4393_end_0 = const()[name = string("op_4393_end_0"), val = tensor<int32, [4]>([32, 3, 1, 1])];
+            tensor<bool, [4]> var_4393_end_mask_0 = const()[name = string("op_4393_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4393_squeeze_mask_0 = const()[name = string("op_4393_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4393_cast_fp16 = slice_by_index(begin = var_4393_begin_0, end = var_4393_end_0, end_mask = var_4393_end_mask_0, squeeze_mask = var_4393_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_4393_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4394_cast_fp16 = mul(x = state_165_cast_fp16, y = var_4393_cast_fp16)[name = string("op_4394_cast_fp16")];
+            tensor<int32, [4]> var_4396_begin_0 = const()[name = string("op_4396_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_4396_end_0 = const()[name = string("op_4396_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_4396_end_mask_0 = const()[name = string("op_4396_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4396_squeeze_mask_0 = const()[name = string("op_4396_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4396_cast_fp16 = slice_by_index(begin = var_4396_begin_0, end = var_4396_end_0, end_mask = var_4396_end_mask_0, squeeze_mask = var_4396_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_4396_cast_fp16")];
+            bool var_4398_transpose_x_1 = const()[name = string("op_4398_transpose_x_1"), val = bool(true)];
+            bool var_4398_transpose_y_1 = const()[name = string("op_4398_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4398_cast_fp16 = matmul(transpose_x = var_4398_transpose_x_1, transpose_y = var_4398_transpose_y_1, x = var_4396_cast_fp16, y = v_new_165_cast_fp16)[name = string("op_4398_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_167_cast_fp16 = add(x = var_4394_cast_fp16, y = var_4398_cast_fp16)[name = string("state_167_cast_fp16")];
+            tensor<int32, [4]> var_4401_begin_0 = const()[name = string("op_4401_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_4401_end_0 = const()[name = string("op_4401_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_4401_end_mask_0 = const()[name = string("op_4401_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4401_squeeze_mask_0 = const()[name = string("op_4401_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4401_cast_fp16 = slice_by_index(begin = var_4401_begin_0, end = var_4401_end_0, end_mask = var_4401_end_mask_0, squeeze_mask = var_4401_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_4401_cast_fp16")];
+            tensor<int32, [4]> var_4403_begin_0 = const()[name = string("op_4403_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_4403_end_0 = const()[name = string("op_4403_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_4403_end_mask_0 = const()[name = string("op_4403_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4403_squeeze_mask_0 = const()[name = string("op_4403_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4403_cast_fp16 = slice_by_index(begin = var_4403_begin_0, end = var_4403_end_0, end_mask = var_4403_end_mask_0, squeeze_mask = var_4403_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_4403_cast_fp16")];
+            bool var_4404_transpose_x_0 = const()[name = string("op_4404_transpose_x_0"), val = bool(false)];
+            bool var_4404_transpose_y_0 = const()[name = string("op_4404_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4404_cast_fp16 = matmul(transpose_x = var_4404_transpose_x_0, transpose_y = var_4404_transpose_y_0, x = var_4403_cast_fp16, y = state_167_cast_fp16)[name = string("op_4404_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_167_cast_fp16 = sub(x = var_4401_cast_fp16, y = var_4404_cast_fp16)[name = string("v_new_167_cast_fp16")];
+            tensor<int32, [4]> var_4407_begin_0 = const()[name = string("op_4407_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_4407_end_0 = const()[name = string("op_4407_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_4407_end_mask_0 = const()[name = string("op_4407_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4407_squeeze_mask_0 = const()[name = string("op_4407_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4407_cast_fp16 = slice_by_index(begin = var_4407_begin_0, end = var_4407_end_0, end_mask = var_4407_end_mask_0, squeeze_mask = var_4407_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_4407_cast_fp16")];
+            bool var_4408_transpose_x_0 = const()[name = string("op_4408_transpose_x_0"), val = bool(false)];
+            bool var_4408_transpose_y_0 = const()[name = string("op_4408_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4408_cast_fp16 = matmul(transpose_x = var_4408_transpose_x_0, transpose_y = var_4408_transpose_y_0, x = var_4407_cast_fp16, y = state_167_cast_fp16)[name = string("op_4408_cast_fp16")];
+            tensor<int32, [4]> var_4410_begin_0 = const()[name = string("op_4410_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_4410_end_0 = const()[name = string("op_4410_end_0"), val = tensor<int32, [4]>([32, 4, 64, 64])];
+            tensor<bool, [4]> var_4410_end_mask_0 = const()[name = string("op_4410_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4410_squeeze_mask_0 = const()[name = string("op_4410_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4410_cast_fp16 = slice_by_index(begin = var_4410_begin_0, end = var_4410_end_0, end_mask = var_4410_end_mask_0, squeeze_mask = var_4410_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_4410_cast_fp16")];
+            bool var_4411_transpose_x_0 = const()[name = string("op_4411_transpose_x_0"), val = bool(false)];
+            bool var_4411_transpose_y_0 = const()[name = string("op_4411_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4411_cast_fp16 = matmul(transpose_x = var_4411_transpose_x_0, transpose_y = var_4411_transpose_y_0, x = var_4410_cast_fp16, y = v_new_167_cast_fp16)[name = string("op_4411_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4412_cast_fp16 = add(x = var_4408_cast_fp16, y = var_4411_cast_fp16)[name = string("op_4412_cast_fp16")];
+            tensor<int32, [4]> var_4414_begin_0 = const()[name = string("op_4414_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_4414_end_0 = const()[name = string("op_4414_end_0"), val = tensor<int32, [4]>([32, 4, 1, 1])];
+            tensor<bool, [4]> var_4414_end_mask_0 = const()[name = string("op_4414_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4414_squeeze_mask_0 = const()[name = string("op_4414_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4414_cast_fp16 = slice_by_index(begin = var_4414_begin_0, end = var_4414_end_0, end_mask = var_4414_end_mask_0, squeeze_mask = var_4414_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_4414_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4415_cast_fp16 = mul(x = state_167_cast_fp16, y = var_4414_cast_fp16)[name = string("op_4415_cast_fp16")];
+            tensor<int32, [4]> var_4417_begin_0 = const()[name = string("op_4417_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_4417_end_0 = const()[name = string("op_4417_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_4417_end_mask_0 = const()[name = string("op_4417_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4417_squeeze_mask_0 = const()[name = string("op_4417_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4417_cast_fp16 = slice_by_index(begin = var_4417_begin_0, end = var_4417_end_0, end_mask = var_4417_end_mask_0, squeeze_mask = var_4417_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_4417_cast_fp16")];
+            bool var_4419_transpose_x_1 = const()[name = string("op_4419_transpose_x_1"), val = bool(true)];
+            bool var_4419_transpose_y_1 = const()[name = string("op_4419_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4419_cast_fp16 = matmul(transpose_x = var_4419_transpose_x_1, transpose_y = var_4419_transpose_y_1, x = var_4417_cast_fp16, y = v_new_167_cast_fp16)[name = string("op_4419_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_169_cast_fp16 = add(x = var_4415_cast_fp16, y = var_4419_cast_fp16)[name = string("state_169_cast_fp16")];
+            tensor<int32, [4]> var_4422_begin_0 = const()[name = string("op_4422_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_4422_end_0 = const()[name = string("op_4422_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_4422_end_mask_0 = const()[name = string("op_4422_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4422_squeeze_mask_0 = const()[name = string("op_4422_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4422_cast_fp16 = slice_by_index(begin = var_4422_begin_0, end = var_4422_end_0, end_mask = var_4422_end_mask_0, squeeze_mask = var_4422_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_4422_cast_fp16")];
+            tensor<int32, [4]> var_4424_begin_0 = const()[name = string("op_4424_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_4424_end_0 = const()[name = string("op_4424_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_4424_end_mask_0 = const()[name = string("op_4424_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4424_squeeze_mask_0 = const()[name = string("op_4424_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4424_cast_fp16 = slice_by_index(begin = var_4424_begin_0, end = var_4424_end_0, end_mask = var_4424_end_mask_0, squeeze_mask = var_4424_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_4424_cast_fp16")];
+            bool var_4425_transpose_x_0 = const()[name = string("op_4425_transpose_x_0"), val = bool(false)];
+            bool var_4425_transpose_y_0 = const()[name = string("op_4425_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4425_cast_fp16 = matmul(transpose_x = var_4425_transpose_x_0, transpose_y = var_4425_transpose_y_0, x = var_4424_cast_fp16, y = state_169_cast_fp16)[name = string("op_4425_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_169_cast_fp16 = sub(x = var_4422_cast_fp16, y = var_4425_cast_fp16)[name = string("v_new_169_cast_fp16")];
+            tensor<int32, [4]> var_4428_begin_0 = const()[name = string("op_4428_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_4428_end_0 = const()[name = string("op_4428_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_4428_end_mask_0 = const()[name = string("op_4428_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4428_squeeze_mask_0 = const()[name = string("op_4428_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4428_cast_fp16 = slice_by_index(begin = var_4428_begin_0, end = var_4428_end_0, end_mask = var_4428_end_mask_0, squeeze_mask = var_4428_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_4428_cast_fp16")];
+            bool var_4429_transpose_x_0 = const()[name = string("op_4429_transpose_x_0"), val = bool(false)];
+            bool var_4429_transpose_y_0 = const()[name = string("op_4429_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4429_cast_fp16 = matmul(transpose_x = var_4429_transpose_x_0, transpose_y = var_4429_transpose_y_0, x = var_4428_cast_fp16, y = state_169_cast_fp16)[name = string("op_4429_cast_fp16")];
+            tensor<int32, [4]> var_4431_begin_0 = const()[name = string("op_4431_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_4431_end_0 = const()[name = string("op_4431_end_0"), val = tensor<int32, [4]>([32, 5, 64, 64])];
+            tensor<bool, [4]> var_4431_end_mask_0 = const()[name = string("op_4431_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4431_squeeze_mask_0 = const()[name = string("op_4431_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4431_cast_fp16 = slice_by_index(begin = var_4431_begin_0, end = var_4431_end_0, end_mask = var_4431_end_mask_0, squeeze_mask = var_4431_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_4431_cast_fp16")];
+            bool var_4432_transpose_x_0 = const()[name = string("op_4432_transpose_x_0"), val = bool(false)];
+            bool var_4432_transpose_y_0 = const()[name = string("op_4432_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4432_cast_fp16 = matmul(transpose_x = var_4432_transpose_x_0, transpose_y = var_4432_transpose_y_0, x = var_4431_cast_fp16, y = v_new_169_cast_fp16)[name = string("op_4432_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4433_cast_fp16 = add(x = var_4429_cast_fp16, y = var_4432_cast_fp16)[name = string("op_4433_cast_fp16")];
+            tensor<int32, [4]> var_4435_begin_0 = const()[name = string("op_4435_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_4435_end_0 = const()[name = string("op_4435_end_0"), val = tensor<int32, [4]>([32, 5, 1, 1])];
+            tensor<bool, [4]> var_4435_end_mask_0 = const()[name = string("op_4435_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4435_squeeze_mask_0 = const()[name = string("op_4435_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4435_cast_fp16 = slice_by_index(begin = var_4435_begin_0, end = var_4435_end_0, end_mask = var_4435_end_mask_0, squeeze_mask = var_4435_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_4435_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4436_cast_fp16 = mul(x = state_169_cast_fp16, y = var_4435_cast_fp16)[name = string("op_4436_cast_fp16")];
+            tensor<int32, [4]> var_4438_begin_0 = const()[name = string("op_4438_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_4438_end_0 = const()[name = string("op_4438_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_4438_end_mask_0 = const()[name = string("op_4438_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4438_squeeze_mask_0 = const()[name = string("op_4438_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4438_cast_fp16 = slice_by_index(begin = var_4438_begin_0, end = var_4438_end_0, end_mask = var_4438_end_mask_0, squeeze_mask = var_4438_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_4438_cast_fp16")];
+            bool var_4440_transpose_x_1 = const()[name = string("op_4440_transpose_x_1"), val = bool(true)];
+            bool var_4440_transpose_y_1 = const()[name = string("op_4440_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4440_cast_fp16 = matmul(transpose_x = var_4440_transpose_x_1, transpose_y = var_4440_transpose_y_1, x = var_4438_cast_fp16, y = v_new_169_cast_fp16)[name = string("op_4440_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_171_cast_fp16 = add(x = var_4436_cast_fp16, y = var_4440_cast_fp16)[name = string("state_171_cast_fp16")];
+            tensor<int32, [4]> var_4443_begin_0 = const()[name = string("op_4443_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_4443_end_0 = const()[name = string("op_4443_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_4443_end_mask_0 = const()[name = string("op_4443_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4443_squeeze_mask_0 = const()[name = string("op_4443_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4443_cast_fp16 = slice_by_index(begin = var_4443_begin_0, end = var_4443_end_0, end_mask = var_4443_end_mask_0, squeeze_mask = var_4443_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_4443_cast_fp16")];
+            tensor<int32, [4]> var_4445_begin_0 = const()[name = string("op_4445_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_4445_end_0 = const()[name = string("op_4445_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_4445_end_mask_0 = const()[name = string("op_4445_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4445_squeeze_mask_0 = const()[name = string("op_4445_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4445_cast_fp16 = slice_by_index(begin = var_4445_begin_0, end = var_4445_end_0, end_mask = var_4445_end_mask_0, squeeze_mask = var_4445_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_4445_cast_fp16")];
+            bool var_4446_transpose_x_0 = const()[name = string("op_4446_transpose_x_0"), val = bool(false)];
+            bool var_4446_transpose_y_0 = const()[name = string("op_4446_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4446_cast_fp16 = matmul(transpose_x = var_4446_transpose_x_0, transpose_y = var_4446_transpose_y_0, x = var_4445_cast_fp16, y = state_171_cast_fp16)[name = string("op_4446_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_171_cast_fp16 = sub(x = var_4443_cast_fp16, y = var_4446_cast_fp16)[name = string("v_new_171_cast_fp16")];
+            tensor<int32, [4]> var_4449_begin_0 = const()[name = string("op_4449_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_4449_end_0 = const()[name = string("op_4449_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_4449_end_mask_0 = const()[name = string("op_4449_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4449_squeeze_mask_0 = const()[name = string("op_4449_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4449_cast_fp16 = slice_by_index(begin = var_4449_begin_0, end = var_4449_end_0, end_mask = var_4449_end_mask_0, squeeze_mask = var_4449_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_4449_cast_fp16")];
+            bool var_4450_transpose_x_0 = const()[name = string("op_4450_transpose_x_0"), val = bool(false)];
+            bool var_4450_transpose_y_0 = const()[name = string("op_4450_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4450_cast_fp16 = matmul(transpose_x = var_4450_transpose_x_0, transpose_y = var_4450_transpose_y_0, x = var_4449_cast_fp16, y = state_171_cast_fp16)[name = string("op_4450_cast_fp16")];
+            tensor<int32, [4]> var_4452_begin_0 = const()[name = string("op_4452_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_4452_end_0 = const()[name = string("op_4452_end_0"), val = tensor<int32, [4]>([32, 6, 64, 64])];
+            tensor<bool, [4]> var_4452_end_mask_0 = const()[name = string("op_4452_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4452_squeeze_mask_0 = const()[name = string("op_4452_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4452_cast_fp16 = slice_by_index(begin = var_4452_begin_0, end = var_4452_end_0, end_mask = var_4452_end_mask_0, squeeze_mask = var_4452_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_4452_cast_fp16")];
+            bool var_4453_transpose_x_0 = const()[name = string("op_4453_transpose_x_0"), val = bool(false)];
+            bool var_4453_transpose_y_0 = const()[name = string("op_4453_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4453_cast_fp16 = matmul(transpose_x = var_4453_transpose_x_0, transpose_y = var_4453_transpose_y_0, x = var_4452_cast_fp16, y = v_new_171_cast_fp16)[name = string("op_4453_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4454_cast_fp16 = add(x = var_4450_cast_fp16, y = var_4453_cast_fp16)[name = string("op_4454_cast_fp16")];
+            tensor<int32, [4]> var_4456_begin_0 = const()[name = string("op_4456_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_4456_end_0 = const()[name = string("op_4456_end_0"), val = tensor<int32, [4]>([32, 6, 1, 1])];
+            tensor<bool, [4]> var_4456_end_mask_0 = const()[name = string("op_4456_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4456_squeeze_mask_0 = const()[name = string("op_4456_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4456_cast_fp16 = slice_by_index(begin = var_4456_begin_0, end = var_4456_end_0, end_mask = var_4456_end_mask_0, squeeze_mask = var_4456_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_4456_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4457_cast_fp16 = mul(x = state_171_cast_fp16, y = var_4456_cast_fp16)[name = string("op_4457_cast_fp16")];
+            tensor<int32, [4]> var_4459_begin_0 = const()[name = string("op_4459_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_4459_end_0 = const()[name = string("op_4459_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_4459_end_mask_0 = const()[name = string("op_4459_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4459_squeeze_mask_0 = const()[name = string("op_4459_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4459_cast_fp16 = slice_by_index(begin = var_4459_begin_0, end = var_4459_end_0, end_mask = var_4459_end_mask_0, squeeze_mask = var_4459_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_4459_cast_fp16")];
+            bool var_4461_transpose_x_1 = const()[name = string("op_4461_transpose_x_1"), val = bool(true)];
+            bool var_4461_transpose_y_1 = const()[name = string("op_4461_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4461_cast_fp16 = matmul(transpose_x = var_4461_transpose_x_1, transpose_y = var_4461_transpose_y_1, x = var_4459_cast_fp16, y = v_new_171_cast_fp16)[name = string("op_4461_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_173_cast_fp16 = add(x = var_4457_cast_fp16, y = var_4461_cast_fp16)[name = string("state_173_cast_fp16")];
+            tensor<int32, [4]> var_4464_begin_0 = const()[name = string("op_4464_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_4464_end_0 = const()[name = string("op_4464_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_4464_end_mask_0 = const()[name = string("op_4464_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4464_squeeze_mask_0 = const()[name = string("op_4464_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4464_cast_fp16 = slice_by_index(begin = var_4464_begin_0, end = var_4464_end_0, end_mask = var_4464_end_mask_0, squeeze_mask = var_4464_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_4464_cast_fp16")];
+            tensor<int32, [4]> var_4466_begin_0 = const()[name = string("op_4466_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_4466_end_0 = const()[name = string("op_4466_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_4466_end_mask_0 = const()[name = string("op_4466_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4466_squeeze_mask_0 = const()[name = string("op_4466_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4466_cast_fp16 = slice_by_index(begin = var_4466_begin_0, end = var_4466_end_0, end_mask = var_4466_end_mask_0, squeeze_mask = var_4466_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_4466_cast_fp16")];
+            bool var_4467_transpose_x_0 = const()[name = string("op_4467_transpose_x_0"), val = bool(false)];
+            bool var_4467_transpose_y_0 = const()[name = string("op_4467_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4467_cast_fp16 = matmul(transpose_x = var_4467_transpose_x_0, transpose_y = var_4467_transpose_y_0, x = var_4466_cast_fp16, y = state_173_cast_fp16)[name = string("op_4467_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_173_cast_fp16 = sub(x = var_4464_cast_fp16, y = var_4467_cast_fp16)[name = string("v_new_173_cast_fp16")];
+            tensor<int32, [4]> var_4470_begin_0 = const()[name = string("op_4470_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_4470_end_0 = const()[name = string("op_4470_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_4470_end_mask_0 = const()[name = string("op_4470_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4470_squeeze_mask_0 = const()[name = string("op_4470_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4470_cast_fp16 = slice_by_index(begin = var_4470_begin_0, end = var_4470_end_0, end_mask = var_4470_end_mask_0, squeeze_mask = var_4470_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_4470_cast_fp16")];
+            bool var_4471_transpose_x_0 = const()[name = string("op_4471_transpose_x_0"), val = bool(false)];
+            bool var_4471_transpose_y_0 = const()[name = string("op_4471_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4471_cast_fp16 = matmul(transpose_x = var_4471_transpose_x_0, transpose_y = var_4471_transpose_y_0, x = var_4470_cast_fp16, y = state_173_cast_fp16)[name = string("op_4471_cast_fp16")];
+            tensor<int32, [4]> var_4473_begin_0 = const()[name = string("op_4473_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_4473_end_0 = const()[name = string("op_4473_end_0"), val = tensor<int32, [4]>([32, 7, 64, 64])];
+            tensor<bool, [4]> var_4473_end_mask_0 = const()[name = string("op_4473_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4473_squeeze_mask_0 = const()[name = string("op_4473_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4473_cast_fp16 = slice_by_index(begin = var_4473_begin_0, end = var_4473_end_0, end_mask = var_4473_end_mask_0, squeeze_mask = var_4473_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_4473_cast_fp16")];
+            bool var_4474_transpose_x_0 = const()[name = string("op_4474_transpose_x_0"), val = bool(false)];
+            bool var_4474_transpose_y_0 = const()[name = string("op_4474_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4474_cast_fp16 = matmul(transpose_x = var_4474_transpose_x_0, transpose_y = var_4474_transpose_y_0, x = var_4473_cast_fp16, y = v_new_173_cast_fp16)[name = string("op_4474_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4475_cast_fp16 = add(x = var_4471_cast_fp16, y = var_4474_cast_fp16)[name = string("op_4475_cast_fp16")];
+            tensor<int32, [4]> var_4477_begin_0 = const()[name = string("op_4477_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_4477_end_0 = const()[name = string("op_4477_end_0"), val = tensor<int32, [4]>([32, 7, 1, 1])];
+            tensor<bool, [4]> var_4477_end_mask_0 = const()[name = string("op_4477_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4477_squeeze_mask_0 = const()[name = string("op_4477_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4477_cast_fp16 = slice_by_index(begin = var_4477_begin_0, end = var_4477_end_0, end_mask = var_4477_end_mask_0, squeeze_mask = var_4477_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_4477_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4478_cast_fp16 = mul(x = state_173_cast_fp16, y = var_4477_cast_fp16)[name = string("op_4478_cast_fp16")];
+            tensor<int32, [4]> var_4480_begin_0 = const()[name = string("op_4480_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_4480_end_0 = const()[name = string("op_4480_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_4480_end_mask_0 = const()[name = string("op_4480_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4480_squeeze_mask_0 = const()[name = string("op_4480_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4480_cast_fp16 = slice_by_index(begin = var_4480_begin_0, end = var_4480_end_0, end_mask = var_4480_end_mask_0, squeeze_mask = var_4480_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_4480_cast_fp16")];
+            bool var_4482_transpose_x_1 = const()[name = string("op_4482_transpose_x_1"), val = bool(true)];
+            bool var_4482_transpose_y_1 = const()[name = string("op_4482_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4482_cast_fp16 = matmul(transpose_x = var_4482_transpose_x_1, transpose_y = var_4482_transpose_y_1, x = var_4480_cast_fp16, y = v_new_173_cast_fp16)[name = string("op_4482_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_175_cast_fp16 = add(x = var_4478_cast_fp16, y = var_4482_cast_fp16)[name = string("state_175_cast_fp16")];
+            tensor<int32, [4]> var_4485_begin_0 = const()[name = string("op_4485_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_4485_end_0 = const()[name = string("op_4485_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_4485_end_mask_0 = const()[name = string("op_4485_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4485_squeeze_mask_0 = const()[name = string("op_4485_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4485_cast_fp16 = slice_by_index(begin = var_4485_begin_0, end = var_4485_end_0, end_mask = var_4485_end_mask_0, squeeze_mask = var_4485_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_4485_cast_fp16")];
+            tensor<int32, [4]> var_4487_begin_0 = const()[name = string("op_4487_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_4487_end_0 = const()[name = string("op_4487_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_4487_end_mask_0 = const()[name = string("op_4487_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4487_squeeze_mask_0 = const()[name = string("op_4487_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4487_cast_fp16 = slice_by_index(begin = var_4487_begin_0, end = var_4487_end_0, end_mask = var_4487_end_mask_0, squeeze_mask = var_4487_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_4487_cast_fp16")];
+            bool var_4488_transpose_x_0 = const()[name = string("op_4488_transpose_x_0"), val = bool(false)];
+            bool var_4488_transpose_y_0 = const()[name = string("op_4488_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4488_cast_fp16 = matmul(transpose_x = var_4488_transpose_x_0, transpose_y = var_4488_transpose_y_0, x = var_4487_cast_fp16, y = state_175_cast_fp16)[name = string("op_4488_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_175_cast_fp16 = sub(x = var_4485_cast_fp16, y = var_4488_cast_fp16)[name = string("v_new_175_cast_fp16")];
+            tensor<int32, [4]> var_4491_begin_0 = const()[name = string("op_4491_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_4491_end_0 = const()[name = string("op_4491_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_4491_end_mask_0 = const()[name = string("op_4491_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4491_squeeze_mask_0 = const()[name = string("op_4491_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4491_cast_fp16 = slice_by_index(begin = var_4491_begin_0, end = var_4491_end_0, end_mask = var_4491_end_mask_0, squeeze_mask = var_4491_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_4491_cast_fp16")];
+            bool var_4492_transpose_x_0 = const()[name = string("op_4492_transpose_x_0"), val = bool(false)];
+            bool var_4492_transpose_y_0 = const()[name = string("op_4492_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4492_cast_fp16 = matmul(transpose_x = var_4492_transpose_x_0, transpose_y = var_4492_transpose_y_0, x = var_4491_cast_fp16, y = state_175_cast_fp16)[name = string("op_4492_cast_fp16")];
+            tensor<int32, [4]> var_4494_begin_0 = const()[name = string("op_4494_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_4494_end_0 = const()[name = string("op_4494_end_0"), val = tensor<int32, [4]>([32, 8, 64, 64])];
+            tensor<bool, [4]> var_4494_end_mask_0 = const()[name = string("op_4494_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4494_squeeze_mask_0 = const()[name = string("op_4494_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4494_cast_fp16 = slice_by_index(begin = var_4494_begin_0, end = var_4494_end_0, end_mask = var_4494_end_mask_0, squeeze_mask = var_4494_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_4494_cast_fp16")];
+            bool var_4495_transpose_x_0 = const()[name = string("op_4495_transpose_x_0"), val = bool(false)];
+            bool var_4495_transpose_y_0 = const()[name = string("op_4495_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4495_cast_fp16 = matmul(transpose_x = var_4495_transpose_x_0, transpose_y = var_4495_transpose_y_0, x = var_4494_cast_fp16, y = v_new_175_cast_fp16)[name = string("op_4495_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4496_cast_fp16 = add(x = var_4492_cast_fp16, y = var_4495_cast_fp16)[name = string("op_4496_cast_fp16")];
+            tensor<int32, [4]> var_4498_begin_0 = const()[name = string("op_4498_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_4498_end_0 = const()[name = string("op_4498_end_0"), val = tensor<int32, [4]>([32, 8, 1, 1])];
+            tensor<bool, [4]> var_4498_end_mask_0 = const()[name = string("op_4498_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4498_squeeze_mask_0 = const()[name = string("op_4498_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4498_cast_fp16 = slice_by_index(begin = var_4498_begin_0, end = var_4498_end_0, end_mask = var_4498_end_mask_0, squeeze_mask = var_4498_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_4498_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4499_cast_fp16 = mul(x = state_175_cast_fp16, y = var_4498_cast_fp16)[name = string("op_4499_cast_fp16")];
+            tensor<int32, [4]> var_4501_begin_0 = const()[name = string("op_4501_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_4501_end_0 = const()[name = string("op_4501_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_4501_end_mask_0 = const()[name = string("op_4501_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4501_squeeze_mask_0 = const()[name = string("op_4501_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4501_cast_fp16 = slice_by_index(begin = var_4501_begin_0, end = var_4501_end_0, end_mask = var_4501_end_mask_0, squeeze_mask = var_4501_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_4501_cast_fp16")];
+            bool var_4503_transpose_x_1 = const()[name = string("op_4503_transpose_x_1"), val = bool(true)];
+            bool var_4503_transpose_y_1 = const()[name = string("op_4503_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4503_cast_fp16 = matmul(transpose_x = var_4503_transpose_x_1, transpose_y = var_4503_transpose_y_1, x = var_4501_cast_fp16, y = v_new_175_cast_fp16)[name = string("op_4503_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_177_cast_fp16 = add(x = var_4499_cast_fp16, y = var_4503_cast_fp16)[name = string("state_177_cast_fp16")];
+            tensor<int32, [4]> var_4506_begin_0 = const()[name = string("op_4506_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_4506_end_0 = const()[name = string("op_4506_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_4506_end_mask_0 = const()[name = string("op_4506_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4506_squeeze_mask_0 = const()[name = string("op_4506_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4506_cast_fp16 = slice_by_index(begin = var_4506_begin_0, end = var_4506_end_0, end_mask = var_4506_end_mask_0, squeeze_mask = var_4506_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_4506_cast_fp16")];
+            tensor<int32, [4]> var_4508_begin_0 = const()[name = string("op_4508_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_4508_end_0 = const()[name = string("op_4508_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_4508_end_mask_0 = const()[name = string("op_4508_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4508_squeeze_mask_0 = const()[name = string("op_4508_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4508_cast_fp16 = slice_by_index(begin = var_4508_begin_0, end = var_4508_end_0, end_mask = var_4508_end_mask_0, squeeze_mask = var_4508_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_4508_cast_fp16")];
+            bool var_4509_transpose_x_0 = const()[name = string("op_4509_transpose_x_0"), val = bool(false)];
+            bool var_4509_transpose_y_0 = const()[name = string("op_4509_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4509_cast_fp16 = matmul(transpose_x = var_4509_transpose_x_0, transpose_y = var_4509_transpose_y_0, x = var_4508_cast_fp16, y = state_177_cast_fp16)[name = string("op_4509_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_177_cast_fp16 = sub(x = var_4506_cast_fp16, y = var_4509_cast_fp16)[name = string("v_new_177_cast_fp16")];
+            tensor<int32, [4]> var_4512_begin_0 = const()[name = string("op_4512_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_4512_end_0 = const()[name = string("op_4512_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_4512_end_mask_0 = const()[name = string("op_4512_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4512_squeeze_mask_0 = const()[name = string("op_4512_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4512_cast_fp16 = slice_by_index(begin = var_4512_begin_0, end = var_4512_end_0, end_mask = var_4512_end_mask_0, squeeze_mask = var_4512_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_4512_cast_fp16")];
+            bool var_4513_transpose_x_0 = const()[name = string("op_4513_transpose_x_0"), val = bool(false)];
+            bool var_4513_transpose_y_0 = const()[name = string("op_4513_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4513_cast_fp16 = matmul(transpose_x = var_4513_transpose_x_0, transpose_y = var_4513_transpose_y_0, x = var_4512_cast_fp16, y = state_177_cast_fp16)[name = string("op_4513_cast_fp16")];
+            tensor<int32, [4]> var_4515_begin_0 = const()[name = string("op_4515_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_4515_end_0 = const()[name = string("op_4515_end_0"), val = tensor<int32, [4]>([32, 9, 64, 64])];
+            tensor<bool, [4]> var_4515_end_mask_0 = const()[name = string("op_4515_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4515_squeeze_mask_0 = const()[name = string("op_4515_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4515_cast_fp16 = slice_by_index(begin = var_4515_begin_0, end = var_4515_end_0, end_mask = var_4515_end_mask_0, squeeze_mask = var_4515_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_4515_cast_fp16")];
+            bool var_4516_transpose_x_0 = const()[name = string("op_4516_transpose_x_0"), val = bool(false)];
+            bool var_4516_transpose_y_0 = const()[name = string("op_4516_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4516_cast_fp16 = matmul(transpose_x = var_4516_transpose_x_0, transpose_y = var_4516_transpose_y_0, x = var_4515_cast_fp16, y = v_new_177_cast_fp16)[name = string("op_4516_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4517_cast_fp16 = add(x = var_4513_cast_fp16, y = var_4516_cast_fp16)[name = string("op_4517_cast_fp16")];
+            tensor<int32, [4]> var_4519_begin_0 = const()[name = string("op_4519_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_4519_end_0 = const()[name = string("op_4519_end_0"), val = tensor<int32, [4]>([32, 9, 1, 1])];
+            tensor<bool, [4]> var_4519_end_mask_0 = const()[name = string("op_4519_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4519_squeeze_mask_0 = const()[name = string("op_4519_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4519_cast_fp16 = slice_by_index(begin = var_4519_begin_0, end = var_4519_end_0, end_mask = var_4519_end_mask_0, squeeze_mask = var_4519_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_4519_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4520_cast_fp16 = mul(x = state_177_cast_fp16, y = var_4519_cast_fp16)[name = string("op_4520_cast_fp16")];
+            tensor<int32, [4]> var_4522_begin_0 = const()[name = string("op_4522_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_4522_end_0 = const()[name = string("op_4522_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_4522_end_mask_0 = const()[name = string("op_4522_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4522_squeeze_mask_0 = const()[name = string("op_4522_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4522_cast_fp16 = slice_by_index(begin = var_4522_begin_0, end = var_4522_end_0, end_mask = var_4522_end_mask_0, squeeze_mask = var_4522_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_4522_cast_fp16")];
+            bool var_4524_transpose_x_1 = const()[name = string("op_4524_transpose_x_1"), val = bool(true)];
+            bool var_4524_transpose_y_1 = const()[name = string("op_4524_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4524_cast_fp16 = matmul(transpose_x = var_4524_transpose_x_1, transpose_y = var_4524_transpose_y_1, x = var_4522_cast_fp16, y = v_new_177_cast_fp16)[name = string("op_4524_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_179_cast_fp16 = add(x = var_4520_cast_fp16, y = var_4524_cast_fp16)[name = string("state_179_cast_fp16")];
+            tensor<int32, [4]> var_4527_begin_0 = const()[name = string("op_4527_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_4527_end_0 = const()[name = string("op_4527_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_4527_end_mask_0 = const()[name = string("op_4527_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4527_squeeze_mask_0 = const()[name = string("op_4527_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4527_cast_fp16 = slice_by_index(begin = var_4527_begin_0, end = var_4527_end_0, end_mask = var_4527_end_mask_0, squeeze_mask = var_4527_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_4527_cast_fp16")];
+            tensor<int32, [4]> var_4529_begin_0 = const()[name = string("op_4529_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_4529_end_0 = const()[name = string("op_4529_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_4529_end_mask_0 = const()[name = string("op_4529_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4529_squeeze_mask_0 = const()[name = string("op_4529_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4529_cast_fp16 = slice_by_index(begin = var_4529_begin_0, end = var_4529_end_0, end_mask = var_4529_end_mask_0, squeeze_mask = var_4529_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_4529_cast_fp16")];
+            bool var_4530_transpose_x_0 = const()[name = string("op_4530_transpose_x_0"), val = bool(false)];
+            bool var_4530_transpose_y_0 = const()[name = string("op_4530_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4530_cast_fp16 = matmul(transpose_x = var_4530_transpose_x_0, transpose_y = var_4530_transpose_y_0, x = var_4529_cast_fp16, y = state_179_cast_fp16)[name = string("op_4530_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_179_cast_fp16 = sub(x = var_4527_cast_fp16, y = var_4530_cast_fp16)[name = string("v_new_179_cast_fp16")];
+            tensor<int32, [4]> var_4533_begin_0 = const()[name = string("op_4533_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_4533_end_0 = const()[name = string("op_4533_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_4533_end_mask_0 = const()[name = string("op_4533_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4533_squeeze_mask_0 = const()[name = string("op_4533_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4533_cast_fp16 = slice_by_index(begin = var_4533_begin_0, end = var_4533_end_0, end_mask = var_4533_end_mask_0, squeeze_mask = var_4533_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_4533_cast_fp16")];
+            bool var_4534_transpose_x_0 = const()[name = string("op_4534_transpose_x_0"), val = bool(false)];
+            bool var_4534_transpose_y_0 = const()[name = string("op_4534_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4534_cast_fp16 = matmul(transpose_x = var_4534_transpose_x_0, transpose_y = var_4534_transpose_y_0, x = var_4533_cast_fp16, y = state_179_cast_fp16)[name = string("op_4534_cast_fp16")];
+            tensor<int32, [4]> var_4536_begin_0 = const()[name = string("op_4536_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_4536_end_0 = const()[name = string("op_4536_end_0"), val = tensor<int32, [4]>([32, 10, 64, 64])];
+            tensor<bool, [4]> var_4536_end_mask_0 = const()[name = string("op_4536_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4536_squeeze_mask_0 = const()[name = string("op_4536_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4536_cast_fp16 = slice_by_index(begin = var_4536_begin_0, end = var_4536_end_0, end_mask = var_4536_end_mask_0, squeeze_mask = var_4536_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_4536_cast_fp16")];
+            bool var_4537_transpose_x_0 = const()[name = string("op_4537_transpose_x_0"), val = bool(false)];
+            bool var_4537_transpose_y_0 = const()[name = string("op_4537_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4537_cast_fp16 = matmul(transpose_x = var_4537_transpose_x_0, transpose_y = var_4537_transpose_y_0, x = var_4536_cast_fp16, y = v_new_179_cast_fp16)[name = string("op_4537_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4538_cast_fp16 = add(x = var_4534_cast_fp16, y = var_4537_cast_fp16)[name = string("op_4538_cast_fp16")];
+            tensor<int32, [4]> var_4540_begin_0 = const()[name = string("op_4540_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_4540_end_0 = const()[name = string("op_4540_end_0"), val = tensor<int32, [4]>([32, 10, 1, 1])];
+            tensor<bool, [4]> var_4540_end_mask_0 = const()[name = string("op_4540_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4540_squeeze_mask_0 = const()[name = string("op_4540_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4540_cast_fp16 = slice_by_index(begin = var_4540_begin_0, end = var_4540_end_0, end_mask = var_4540_end_mask_0, squeeze_mask = var_4540_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_4540_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4541_cast_fp16 = mul(x = state_179_cast_fp16, y = var_4540_cast_fp16)[name = string("op_4541_cast_fp16")];
+            tensor<int32, [4]> var_4543_begin_0 = const()[name = string("op_4543_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_4543_end_0 = const()[name = string("op_4543_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_4543_end_mask_0 = const()[name = string("op_4543_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4543_squeeze_mask_0 = const()[name = string("op_4543_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4543_cast_fp16 = slice_by_index(begin = var_4543_begin_0, end = var_4543_end_0, end_mask = var_4543_end_mask_0, squeeze_mask = var_4543_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_4543_cast_fp16")];
+            bool var_4545_transpose_x_1 = const()[name = string("op_4545_transpose_x_1"), val = bool(true)];
+            bool var_4545_transpose_y_1 = const()[name = string("op_4545_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4545_cast_fp16 = matmul(transpose_x = var_4545_transpose_x_1, transpose_y = var_4545_transpose_y_1, x = var_4543_cast_fp16, y = v_new_179_cast_fp16)[name = string("op_4545_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_181_cast_fp16 = add(x = var_4541_cast_fp16, y = var_4545_cast_fp16)[name = string("state_181_cast_fp16")];
+            tensor<int32, [4]> var_4548_begin_0 = const()[name = string("op_4548_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_4548_end_0 = const()[name = string("op_4548_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_4548_end_mask_0 = const()[name = string("op_4548_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4548_squeeze_mask_0 = const()[name = string("op_4548_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4548_cast_fp16 = slice_by_index(begin = var_4548_begin_0, end = var_4548_end_0, end_mask = var_4548_end_mask_0, squeeze_mask = var_4548_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_4548_cast_fp16")];
+            tensor<int32, [4]> var_4550_begin_0 = const()[name = string("op_4550_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_4550_end_0 = const()[name = string("op_4550_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_4550_end_mask_0 = const()[name = string("op_4550_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4550_squeeze_mask_0 = const()[name = string("op_4550_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4550_cast_fp16 = slice_by_index(begin = var_4550_begin_0, end = var_4550_end_0, end_mask = var_4550_end_mask_0, squeeze_mask = var_4550_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_4550_cast_fp16")];
+            bool var_4551_transpose_x_0 = const()[name = string("op_4551_transpose_x_0"), val = bool(false)];
+            bool var_4551_transpose_y_0 = const()[name = string("op_4551_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4551_cast_fp16 = matmul(transpose_x = var_4551_transpose_x_0, transpose_y = var_4551_transpose_y_0, x = var_4550_cast_fp16, y = state_181_cast_fp16)[name = string("op_4551_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_181_cast_fp16 = sub(x = var_4548_cast_fp16, y = var_4551_cast_fp16)[name = string("v_new_181_cast_fp16")];
+            tensor<int32, [4]> var_4554_begin_0 = const()[name = string("op_4554_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_4554_end_0 = const()[name = string("op_4554_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_4554_end_mask_0 = const()[name = string("op_4554_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4554_squeeze_mask_0 = const()[name = string("op_4554_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4554_cast_fp16 = slice_by_index(begin = var_4554_begin_0, end = var_4554_end_0, end_mask = var_4554_end_mask_0, squeeze_mask = var_4554_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_4554_cast_fp16")];
+            bool var_4555_transpose_x_0 = const()[name = string("op_4555_transpose_x_0"), val = bool(false)];
+            bool var_4555_transpose_y_0 = const()[name = string("op_4555_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4555_cast_fp16 = matmul(transpose_x = var_4555_transpose_x_0, transpose_y = var_4555_transpose_y_0, x = var_4554_cast_fp16, y = state_181_cast_fp16)[name = string("op_4555_cast_fp16")];
+            tensor<int32, [4]> var_4557_begin_0 = const()[name = string("op_4557_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_4557_end_0 = const()[name = string("op_4557_end_0"), val = tensor<int32, [4]>([32, 11, 64, 64])];
+            tensor<bool, [4]> var_4557_end_mask_0 = const()[name = string("op_4557_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4557_squeeze_mask_0 = const()[name = string("op_4557_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4557_cast_fp16 = slice_by_index(begin = var_4557_begin_0, end = var_4557_end_0, end_mask = var_4557_end_mask_0, squeeze_mask = var_4557_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_4557_cast_fp16")];
+            bool var_4558_transpose_x_0 = const()[name = string("op_4558_transpose_x_0"), val = bool(false)];
+            bool var_4558_transpose_y_0 = const()[name = string("op_4558_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4558_cast_fp16 = matmul(transpose_x = var_4558_transpose_x_0, transpose_y = var_4558_transpose_y_0, x = var_4557_cast_fp16, y = v_new_181_cast_fp16)[name = string("op_4558_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4559_cast_fp16 = add(x = var_4555_cast_fp16, y = var_4558_cast_fp16)[name = string("op_4559_cast_fp16")];
+            tensor<int32, [4]> var_4561_begin_0 = const()[name = string("op_4561_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_4561_end_0 = const()[name = string("op_4561_end_0"), val = tensor<int32, [4]>([32, 11, 1, 1])];
+            tensor<bool, [4]> var_4561_end_mask_0 = const()[name = string("op_4561_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4561_squeeze_mask_0 = const()[name = string("op_4561_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4561_cast_fp16 = slice_by_index(begin = var_4561_begin_0, end = var_4561_end_0, end_mask = var_4561_end_mask_0, squeeze_mask = var_4561_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_4561_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4562_cast_fp16 = mul(x = state_181_cast_fp16, y = var_4561_cast_fp16)[name = string("op_4562_cast_fp16")];
+            tensor<int32, [4]> var_4564_begin_0 = const()[name = string("op_4564_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_4564_end_0 = const()[name = string("op_4564_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_4564_end_mask_0 = const()[name = string("op_4564_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4564_squeeze_mask_0 = const()[name = string("op_4564_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4564_cast_fp16 = slice_by_index(begin = var_4564_begin_0, end = var_4564_end_0, end_mask = var_4564_end_mask_0, squeeze_mask = var_4564_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_4564_cast_fp16")];
+            bool var_4566_transpose_x_1 = const()[name = string("op_4566_transpose_x_1"), val = bool(true)];
+            bool var_4566_transpose_y_1 = const()[name = string("op_4566_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4566_cast_fp16 = matmul(transpose_x = var_4566_transpose_x_1, transpose_y = var_4566_transpose_y_1, x = var_4564_cast_fp16, y = v_new_181_cast_fp16)[name = string("op_4566_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_183_cast_fp16 = add(x = var_4562_cast_fp16, y = var_4566_cast_fp16)[name = string("state_183_cast_fp16")];
+            tensor<int32, [4]> var_4569_begin_0 = const()[name = string("op_4569_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_4569_end_0 = const()[name = string("op_4569_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_4569_end_mask_0 = const()[name = string("op_4569_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4569_squeeze_mask_0 = const()[name = string("op_4569_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4569_cast_fp16 = slice_by_index(begin = var_4569_begin_0, end = var_4569_end_0, end_mask = var_4569_end_mask_0, squeeze_mask = var_4569_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_4569_cast_fp16")];
+            tensor<int32, [4]> var_4571_begin_0 = const()[name = string("op_4571_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_4571_end_0 = const()[name = string("op_4571_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_4571_end_mask_0 = const()[name = string("op_4571_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4571_squeeze_mask_0 = const()[name = string("op_4571_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4571_cast_fp16 = slice_by_index(begin = var_4571_begin_0, end = var_4571_end_0, end_mask = var_4571_end_mask_0, squeeze_mask = var_4571_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_4571_cast_fp16")];
+            bool var_4572_transpose_x_0 = const()[name = string("op_4572_transpose_x_0"), val = bool(false)];
+            bool var_4572_transpose_y_0 = const()[name = string("op_4572_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4572_cast_fp16 = matmul(transpose_x = var_4572_transpose_x_0, transpose_y = var_4572_transpose_y_0, x = var_4571_cast_fp16, y = state_183_cast_fp16)[name = string("op_4572_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_183_cast_fp16 = sub(x = var_4569_cast_fp16, y = var_4572_cast_fp16)[name = string("v_new_183_cast_fp16")];
+            tensor<int32, [4]> var_4575_begin_0 = const()[name = string("op_4575_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_4575_end_0 = const()[name = string("op_4575_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_4575_end_mask_0 = const()[name = string("op_4575_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4575_squeeze_mask_0 = const()[name = string("op_4575_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4575_cast_fp16 = slice_by_index(begin = var_4575_begin_0, end = var_4575_end_0, end_mask = var_4575_end_mask_0, squeeze_mask = var_4575_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_4575_cast_fp16")];
+            bool var_4576_transpose_x_0 = const()[name = string("op_4576_transpose_x_0"), val = bool(false)];
+            bool var_4576_transpose_y_0 = const()[name = string("op_4576_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4576_cast_fp16 = matmul(transpose_x = var_4576_transpose_x_0, transpose_y = var_4576_transpose_y_0, x = var_4575_cast_fp16, y = state_183_cast_fp16)[name = string("op_4576_cast_fp16")];
+            tensor<int32, [4]> var_4578_begin_0 = const()[name = string("op_4578_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_4578_end_0 = const()[name = string("op_4578_end_0"), val = tensor<int32, [4]>([32, 12, 64, 64])];
+            tensor<bool, [4]> var_4578_end_mask_0 = const()[name = string("op_4578_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4578_squeeze_mask_0 = const()[name = string("op_4578_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4578_cast_fp16 = slice_by_index(begin = var_4578_begin_0, end = var_4578_end_0, end_mask = var_4578_end_mask_0, squeeze_mask = var_4578_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_4578_cast_fp16")];
+            bool var_4579_transpose_x_0 = const()[name = string("op_4579_transpose_x_0"), val = bool(false)];
+            bool var_4579_transpose_y_0 = const()[name = string("op_4579_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4579_cast_fp16 = matmul(transpose_x = var_4579_transpose_x_0, transpose_y = var_4579_transpose_y_0, x = var_4578_cast_fp16, y = v_new_183_cast_fp16)[name = string("op_4579_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4580_cast_fp16 = add(x = var_4576_cast_fp16, y = var_4579_cast_fp16)[name = string("op_4580_cast_fp16")];
+            tensor<int32, [4]> var_4582_begin_0 = const()[name = string("op_4582_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_4582_end_0 = const()[name = string("op_4582_end_0"), val = tensor<int32, [4]>([32, 12, 1, 1])];
+            tensor<bool, [4]> var_4582_end_mask_0 = const()[name = string("op_4582_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4582_squeeze_mask_0 = const()[name = string("op_4582_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4582_cast_fp16 = slice_by_index(begin = var_4582_begin_0, end = var_4582_end_0, end_mask = var_4582_end_mask_0, squeeze_mask = var_4582_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_4582_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4583_cast_fp16 = mul(x = state_183_cast_fp16, y = var_4582_cast_fp16)[name = string("op_4583_cast_fp16")];
+            tensor<int32, [4]> var_4585_begin_0 = const()[name = string("op_4585_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_4585_end_0 = const()[name = string("op_4585_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_4585_end_mask_0 = const()[name = string("op_4585_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4585_squeeze_mask_0 = const()[name = string("op_4585_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4585_cast_fp16 = slice_by_index(begin = var_4585_begin_0, end = var_4585_end_0, end_mask = var_4585_end_mask_0, squeeze_mask = var_4585_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_4585_cast_fp16")];
+            bool var_4587_transpose_x_1 = const()[name = string("op_4587_transpose_x_1"), val = bool(true)];
+            bool var_4587_transpose_y_1 = const()[name = string("op_4587_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4587_cast_fp16 = matmul(transpose_x = var_4587_transpose_x_1, transpose_y = var_4587_transpose_y_1, x = var_4585_cast_fp16, y = v_new_183_cast_fp16)[name = string("op_4587_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_185_cast_fp16 = add(x = var_4583_cast_fp16, y = var_4587_cast_fp16)[name = string("state_185_cast_fp16")];
+            tensor<int32, [4]> var_4590_begin_0 = const()[name = string("op_4590_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_4590_end_0 = const()[name = string("op_4590_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_4590_end_mask_0 = const()[name = string("op_4590_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4590_squeeze_mask_0 = const()[name = string("op_4590_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4590_cast_fp16 = slice_by_index(begin = var_4590_begin_0, end = var_4590_end_0, end_mask = var_4590_end_mask_0, squeeze_mask = var_4590_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_4590_cast_fp16")];
+            tensor<int32, [4]> var_4592_begin_0 = const()[name = string("op_4592_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_4592_end_0 = const()[name = string("op_4592_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_4592_end_mask_0 = const()[name = string("op_4592_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4592_squeeze_mask_0 = const()[name = string("op_4592_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4592_cast_fp16 = slice_by_index(begin = var_4592_begin_0, end = var_4592_end_0, end_mask = var_4592_end_mask_0, squeeze_mask = var_4592_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_4592_cast_fp16")];
+            bool var_4593_transpose_x_0 = const()[name = string("op_4593_transpose_x_0"), val = bool(false)];
+            bool var_4593_transpose_y_0 = const()[name = string("op_4593_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4593_cast_fp16 = matmul(transpose_x = var_4593_transpose_x_0, transpose_y = var_4593_transpose_y_0, x = var_4592_cast_fp16, y = state_185_cast_fp16)[name = string("op_4593_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_185_cast_fp16 = sub(x = var_4590_cast_fp16, y = var_4593_cast_fp16)[name = string("v_new_185_cast_fp16")];
+            tensor<int32, [4]> var_4596_begin_0 = const()[name = string("op_4596_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_4596_end_0 = const()[name = string("op_4596_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_4596_end_mask_0 = const()[name = string("op_4596_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4596_squeeze_mask_0 = const()[name = string("op_4596_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4596_cast_fp16 = slice_by_index(begin = var_4596_begin_0, end = var_4596_end_0, end_mask = var_4596_end_mask_0, squeeze_mask = var_4596_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_4596_cast_fp16")];
+            bool var_4597_transpose_x_0 = const()[name = string("op_4597_transpose_x_0"), val = bool(false)];
+            bool var_4597_transpose_y_0 = const()[name = string("op_4597_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4597_cast_fp16 = matmul(transpose_x = var_4597_transpose_x_0, transpose_y = var_4597_transpose_y_0, x = var_4596_cast_fp16, y = state_185_cast_fp16)[name = string("op_4597_cast_fp16")];
+            tensor<int32, [4]> var_4599_begin_0 = const()[name = string("op_4599_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_4599_end_0 = const()[name = string("op_4599_end_0"), val = tensor<int32, [4]>([32, 13, 64, 64])];
+            tensor<bool, [4]> var_4599_end_mask_0 = const()[name = string("op_4599_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4599_squeeze_mask_0 = const()[name = string("op_4599_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4599_cast_fp16 = slice_by_index(begin = var_4599_begin_0, end = var_4599_end_0, end_mask = var_4599_end_mask_0, squeeze_mask = var_4599_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_4599_cast_fp16")];
+            bool var_4600_transpose_x_0 = const()[name = string("op_4600_transpose_x_0"), val = bool(false)];
+            bool var_4600_transpose_y_0 = const()[name = string("op_4600_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4600_cast_fp16 = matmul(transpose_x = var_4600_transpose_x_0, transpose_y = var_4600_transpose_y_0, x = var_4599_cast_fp16, y = v_new_185_cast_fp16)[name = string("op_4600_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4601_cast_fp16 = add(x = var_4597_cast_fp16, y = var_4600_cast_fp16)[name = string("op_4601_cast_fp16")];
+            tensor<int32, [4]> var_4603_begin_0 = const()[name = string("op_4603_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_4603_end_0 = const()[name = string("op_4603_end_0"), val = tensor<int32, [4]>([32, 13, 1, 1])];
+            tensor<bool, [4]> var_4603_end_mask_0 = const()[name = string("op_4603_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4603_squeeze_mask_0 = const()[name = string("op_4603_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4603_cast_fp16 = slice_by_index(begin = var_4603_begin_0, end = var_4603_end_0, end_mask = var_4603_end_mask_0, squeeze_mask = var_4603_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_4603_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4604_cast_fp16 = mul(x = state_185_cast_fp16, y = var_4603_cast_fp16)[name = string("op_4604_cast_fp16")];
+            tensor<int32, [4]> var_4606_begin_0 = const()[name = string("op_4606_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_4606_end_0 = const()[name = string("op_4606_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_4606_end_mask_0 = const()[name = string("op_4606_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4606_squeeze_mask_0 = const()[name = string("op_4606_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4606_cast_fp16 = slice_by_index(begin = var_4606_begin_0, end = var_4606_end_0, end_mask = var_4606_end_mask_0, squeeze_mask = var_4606_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_4606_cast_fp16")];
+            bool var_4608_transpose_x_1 = const()[name = string("op_4608_transpose_x_1"), val = bool(true)];
+            bool var_4608_transpose_y_1 = const()[name = string("op_4608_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4608_cast_fp16 = matmul(transpose_x = var_4608_transpose_x_1, transpose_y = var_4608_transpose_y_1, x = var_4606_cast_fp16, y = v_new_185_cast_fp16)[name = string("op_4608_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_187_cast_fp16 = add(x = var_4604_cast_fp16, y = var_4608_cast_fp16)[name = string("state_187_cast_fp16")];
+            tensor<int32, [4]> var_4611_begin_0 = const()[name = string("op_4611_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_4611_end_0 = const()[name = string("op_4611_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_4611_end_mask_0 = const()[name = string("op_4611_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4611_squeeze_mask_0 = const()[name = string("op_4611_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4611_cast_fp16 = slice_by_index(begin = var_4611_begin_0, end = var_4611_end_0, end_mask = var_4611_end_mask_0, squeeze_mask = var_4611_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_4611_cast_fp16")];
+            tensor<int32, [4]> var_4613_begin_0 = const()[name = string("op_4613_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_4613_end_0 = const()[name = string("op_4613_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_4613_end_mask_0 = const()[name = string("op_4613_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4613_squeeze_mask_0 = const()[name = string("op_4613_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4613_cast_fp16 = slice_by_index(begin = var_4613_begin_0, end = var_4613_end_0, end_mask = var_4613_end_mask_0, squeeze_mask = var_4613_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_4613_cast_fp16")];
+            bool var_4614_transpose_x_0 = const()[name = string("op_4614_transpose_x_0"), val = bool(false)];
+            bool var_4614_transpose_y_0 = const()[name = string("op_4614_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4614_cast_fp16 = matmul(transpose_x = var_4614_transpose_x_0, transpose_y = var_4614_transpose_y_0, x = var_4613_cast_fp16, y = state_187_cast_fp16)[name = string("op_4614_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_187_cast_fp16 = sub(x = var_4611_cast_fp16, y = var_4614_cast_fp16)[name = string("v_new_187_cast_fp16")];
+            tensor<int32, [4]> var_4617_begin_0 = const()[name = string("op_4617_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_4617_end_0 = const()[name = string("op_4617_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_4617_end_mask_0 = const()[name = string("op_4617_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4617_squeeze_mask_0 = const()[name = string("op_4617_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4617_cast_fp16 = slice_by_index(begin = var_4617_begin_0, end = var_4617_end_0, end_mask = var_4617_end_mask_0, squeeze_mask = var_4617_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_4617_cast_fp16")];
+            bool var_4618_transpose_x_0 = const()[name = string("op_4618_transpose_x_0"), val = bool(false)];
+            bool var_4618_transpose_y_0 = const()[name = string("op_4618_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4618_cast_fp16 = matmul(transpose_x = var_4618_transpose_x_0, transpose_y = var_4618_transpose_y_0, x = var_4617_cast_fp16, y = state_187_cast_fp16)[name = string("op_4618_cast_fp16")];
+            tensor<int32, [4]> var_4620_begin_0 = const()[name = string("op_4620_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_4620_end_0 = const()[name = string("op_4620_end_0"), val = tensor<int32, [4]>([32, 14, 64, 64])];
+            tensor<bool, [4]> var_4620_end_mask_0 = const()[name = string("op_4620_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4620_squeeze_mask_0 = const()[name = string("op_4620_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4620_cast_fp16 = slice_by_index(begin = var_4620_begin_0, end = var_4620_end_0, end_mask = var_4620_end_mask_0, squeeze_mask = var_4620_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_4620_cast_fp16")];
+            bool var_4621_transpose_x_0 = const()[name = string("op_4621_transpose_x_0"), val = bool(false)];
+            bool var_4621_transpose_y_0 = const()[name = string("op_4621_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4621_cast_fp16 = matmul(transpose_x = var_4621_transpose_x_0, transpose_y = var_4621_transpose_y_0, x = var_4620_cast_fp16, y = v_new_187_cast_fp16)[name = string("op_4621_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4622_cast_fp16 = add(x = var_4618_cast_fp16, y = var_4621_cast_fp16)[name = string("op_4622_cast_fp16")];
+            tensor<int32, [4]> var_4624_begin_0 = const()[name = string("op_4624_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_4624_end_0 = const()[name = string("op_4624_end_0"), val = tensor<int32, [4]>([32, 14, 1, 1])];
+            tensor<bool, [4]> var_4624_end_mask_0 = const()[name = string("op_4624_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4624_squeeze_mask_0 = const()[name = string("op_4624_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4624_cast_fp16 = slice_by_index(begin = var_4624_begin_0, end = var_4624_end_0, end_mask = var_4624_end_mask_0, squeeze_mask = var_4624_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_4624_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4625_cast_fp16 = mul(x = state_187_cast_fp16, y = var_4624_cast_fp16)[name = string("op_4625_cast_fp16")];
+            tensor<int32, [4]> var_4627_begin_0 = const()[name = string("op_4627_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_4627_end_0 = const()[name = string("op_4627_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_4627_end_mask_0 = const()[name = string("op_4627_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4627_squeeze_mask_0 = const()[name = string("op_4627_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4627_cast_fp16 = slice_by_index(begin = var_4627_begin_0, end = var_4627_end_0, end_mask = var_4627_end_mask_0, squeeze_mask = var_4627_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_4627_cast_fp16")];
+            bool var_4629_transpose_x_1 = const()[name = string("op_4629_transpose_x_1"), val = bool(true)];
+            bool var_4629_transpose_y_1 = const()[name = string("op_4629_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4629_cast_fp16 = matmul(transpose_x = var_4629_transpose_x_1, transpose_y = var_4629_transpose_y_1, x = var_4627_cast_fp16, y = v_new_187_cast_fp16)[name = string("op_4629_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_189_cast_fp16 = add(x = var_4625_cast_fp16, y = var_4629_cast_fp16)[name = string("state_189_cast_fp16")];
+            tensor<int32, [4]> var_4632_begin_0 = const()[name = string("op_4632_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_4632_end_0 = const()[name = string("op_4632_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_4632_end_mask_0 = const()[name = string("op_4632_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4632_squeeze_mask_0 = const()[name = string("op_4632_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4632_cast_fp16 = slice_by_index(begin = var_4632_begin_0, end = var_4632_end_0, end_mask = var_4632_end_mask_0, squeeze_mask = var_4632_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_4632_cast_fp16")];
+            tensor<int32, [4]> var_4634_begin_0 = const()[name = string("op_4634_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_4634_end_0 = const()[name = string("op_4634_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_4634_end_mask_0 = const()[name = string("op_4634_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4634_squeeze_mask_0 = const()[name = string("op_4634_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4634_cast_fp16 = slice_by_index(begin = var_4634_begin_0, end = var_4634_end_0, end_mask = var_4634_end_mask_0, squeeze_mask = var_4634_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_4634_cast_fp16")];
+            bool var_4635_transpose_x_0 = const()[name = string("op_4635_transpose_x_0"), val = bool(false)];
+            bool var_4635_transpose_y_0 = const()[name = string("op_4635_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4635_cast_fp16 = matmul(transpose_x = var_4635_transpose_x_0, transpose_y = var_4635_transpose_y_0, x = var_4634_cast_fp16, y = state_189_cast_fp16)[name = string("op_4635_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_189_cast_fp16 = sub(x = var_4632_cast_fp16, y = var_4635_cast_fp16)[name = string("v_new_189_cast_fp16")];
+            tensor<int32, [4]> var_4638_begin_0 = const()[name = string("op_4638_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_4638_end_0 = const()[name = string("op_4638_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_4638_end_mask_0 = const()[name = string("op_4638_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4638_squeeze_mask_0 = const()[name = string("op_4638_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4638_cast_fp16 = slice_by_index(begin = var_4638_begin_0, end = var_4638_end_0, end_mask = var_4638_end_mask_0, squeeze_mask = var_4638_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_4638_cast_fp16")];
+            bool var_4639_transpose_x_0 = const()[name = string("op_4639_transpose_x_0"), val = bool(false)];
+            bool var_4639_transpose_y_0 = const()[name = string("op_4639_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4639_cast_fp16 = matmul(transpose_x = var_4639_transpose_x_0, transpose_y = var_4639_transpose_y_0, x = var_4638_cast_fp16, y = state_189_cast_fp16)[name = string("op_4639_cast_fp16")];
+            tensor<int32, [4]> var_4641_begin_0 = const()[name = string("op_4641_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_4641_end_0 = const()[name = string("op_4641_end_0"), val = tensor<int32, [4]>([32, 15, 64, 64])];
+            tensor<bool, [4]> var_4641_end_mask_0 = const()[name = string("op_4641_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4641_squeeze_mask_0 = const()[name = string("op_4641_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4641_cast_fp16 = slice_by_index(begin = var_4641_begin_0, end = var_4641_end_0, end_mask = var_4641_end_mask_0, squeeze_mask = var_4641_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_4641_cast_fp16")];
+            bool var_4642_transpose_x_0 = const()[name = string("op_4642_transpose_x_0"), val = bool(false)];
+            bool var_4642_transpose_y_0 = const()[name = string("op_4642_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4642_cast_fp16 = matmul(transpose_x = var_4642_transpose_x_0, transpose_y = var_4642_transpose_y_0, x = var_4641_cast_fp16, y = v_new_189_cast_fp16)[name = string("op_4642_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4643_cast_fp16 = add(x = var_4639_cast_fp16, y = var_4642_cast_fp16)[name = string("op_4643_cast_fp16")];
+            tensor<int32, [4]> var_4645_begin_0 = const()[name = string("op_4645_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_4645_end_0 = const()[name = string("op_4645_end_0"), val = tensor<int32, [4]>([32, 15, 1, 1])];
+            tensor<bool, [4]> var_4645_end_mask_0 = const()[name = string("op_4645_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4645_squeeze_mask_0 = const()[name = string("op_4645_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4645_cast_fp16 = slice_by_index(begin = var_4645_begin_0, end = var_4645_end_0, end_mask = var_4645_end_mask_0, squeeze_mask = var_4645_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_4645_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4646_cast_fp16 = mul(x = state_189_cast_fp16, y = var_4645_cast_fp16)[name = string("op_4646_cast_fp16")];
+            tensor<int32, [4]> var_4648_begin_0 = const()[name = string("op_4648_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_4648_end_0 = const()[name = string("op_4648_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_4648_end_mask_0 = const()[name = string("op_4648_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4648_squeeze_mask_0 = const()[name = string("op_4648_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4648_cast_fp16 = slice_by_index(begin = var_4648_begin_0, end = var_4648_end_0, end_mask = var_4648_end_mask_0, squeeze_mask = var_4648_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_4648_cast_fp16")];
+            bool var_4650_transpose_x_1 = const()[name = string("op_4650_transpose_x_1"), val = bool(true)];
+            bool var_4650_transpose_y_1 = const()[name = string("op_4650_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4650_cast_fp16 = matmul(transpose_x = var_4650_transpose_x_1, transpose_y = var_4650_transpose_y_1, x = var_4648_cast_fp16, y = v_new_189_cast_fp16)[name = string("op_4650_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_cast_fp16 = add(x = var_4646_cast_fp16, y = var_4650_cast_fp16)[name = string("state_cast_fp16")];
+            tensor<int32, [4]> var_4653_begin_0 = const()[name = string("op_4653_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_4653_end_0 = const()[name = string("op_4653_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_4653_end_mask_0 = const()[name = string("op_4653_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4653_squeeze_mask_0 = const()[name = string("op_4653_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4653_cast_fp16 = slice_by_index(begin = var_4653_begin_0, end = var_4653_end_0, end_mask = var_4653_end_mask_0, squeeze_mask = var_4653_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_4653_cast_fp16")];
+            tensor<int32, [4]> var_4655_begin_0 = const()[name = string("op_4655_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_4655_end_0 = const()[name = string("op_4655_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_4655_end_mask_0 = const()[name = string("op_4655_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4655_squeeze_mask_0 = const()[name = string("op_4655_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4655_cast_fp16 = slice_by_index(begin = var_4655_begin_0, end = var_4655_end_0, end_mask = var_4655_end_mask_0, squeeze_mask = var_4655_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_4655_cast_fp16")];
+            bool var_4656_transpose_x_0 = const()[name = string("op_4656_transpose_x_0"), val = bool(false)];
+            bool var_4656_transpose_y_0 = const()[name = string("op_4656_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4656_cast_fp16 = matmul(transpose_x = var_4656_transpose_x_0, transpose_y = var_4656_transpose_y_0, x = var_4655_cast_fp16, y = state_cast_fp16)[name = string("op_4656_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_cast_fp16 = sub(x = var_4653_cast_fp16, y = var_4656_cast_fp16)[name = string("v_new_cast_fp16")];
+            tensor<int32, [4]> var_4659_begin_0 = const()[name = string("op_4659_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_4659_end_0 = const()[name = string("op_4659_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_4659_end_mask_0 = const()[name = string("op_4659_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4659_squeeze_mask_0 = const()[name = string("op_4659_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4659_cast_fp16 = slice_by_index(begin = var_4659_begin_0, end = var_4659_end_0, end_mask = var_4659_end_mask_0, squeeze_mask = var_4659_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_4659_cast_fp16")];
+            bool var_4660_transpose_x_0 = const()[name = string("op_4660_transpose_x_0"), val = bool(false)];
+            bool var_4660_transpose_y_0 = const()[name = string("op_4660_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4660_cast_fp16 = matmul(transpose_x = var_4660_transpose_x_0, transpose_y = var_4660_transpose_y_0, x = var_4659_cast_fp16, y = state_cast_fp16)[name = string("op_4660_cast_fp16")];
+            tensor<int32, [4]> var_4662_begin_0 = const()[name = string("op_4662_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_4662_end_0 = const()[name = string("op_4662_end_0"), val = tensor<int32, [4]>([32, 16, 64, 64])];
+            tensor<bool, [4]> var_4662_end_mask_0 = const()[name = string("op_4662_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4662_squeeze_mask_0 = const()[name = string("op_4662_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4662_cast_fp16 = slice_by_index(begin = var_4662_begin_0, end = var_4662_end_0, end_mask = var_4662_end_mask_0, squeeze_mask = var_4662_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_4662_cast_fp16")];
+            bool var_4663_transpose_x_0 = const()[name = string("op_4663_transpose_x_0"), val = bool(false)];
+            bool var_4663_transpose_y_0 = const()[name = string("op_4663_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4663_cast_fp16 = matmul(transpose_x = var_4663_transpose_x_0, transpose_y = var_4663_transpose_y_0, x = var_4662_cast_fp16, y = v_new_cast_fp16)[name = string("op_4663_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4664_cast_fp16 = add(x = var_4660_cast_fp16, y = var_4663_cast_fp16)[name = string("op_4664_cast_fp16")];
+            int32 var_4666_axis_0 = const()[name = string("op_4666_axis_0"), val = int32(1)];
+            tensor<fp16, [32, 16, 64, 128]> var_4666_cast_fp16 = stack(axis = var_4666_axis_0, values = (var_4349_cast_fp16, var_4370_cast_fp16, var_4391_cast_fp16, var_4412_cast_fp16, var_4433_cast_fp16, var_4454_cast_fp16, var_4475_cast_fp16, var_4496_cast_fp16, var_4517_cast_fp16, var_4538_cast_fp16, var_4559_cast_fp16, var_4580_cast_fp16, var_4601_cast_fp16, var_4622_cast_fp16, var_4643_cast_fp16, var_4664_cast_fp16))[name = string("op_4666_cast_fp16")];
+            tensor<int32, [3]> var_4667 = const()[name = string("op_4667"), val = tensor<int32, [3]>([32, 1024, 128])];
+            tensor<fp16, [32, 1024, 128]> var_4668_cast_fp16 = reshape(shape = var_4667, x = var_4666_cast_fp16)[name = string("op_4668_cast_fp16")];
+            tensor<int32, [3]> var_4669 = const()[name = string("op_4669"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> input_95_begin_0 = const()[name = string("input_95_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> input_95_end_0 = const()[name = string("input_95_end_0"), val = tensor<int32, [4]>([1, 1024, 32, 128])];
+            tensor<bool, [4]> input_95_end_mask_0 = const()[name = string("input_95_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> input_95_squeeze_mask_0 = const()[name = string("input_95_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [1024, 32, 128]> input_95_cast_fp16 = slice_by_index(begin = input_95_begin_0, end = input_95_end_0, end_mask = input_95_end_mask_0, squeeze_mask = input_95_squeeze_mask_0, x = z_cast_fp16)[name = string("input_95_cast_fp16")];
+            fp16 var_3985_promoted_1_to_fp16 = const()[name = string("op_3985_promoted_1_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1024, 32, 128]> x_169_cast_fp16 = transpose(perm = var_4669, x = var_4668_cast_fp16)[name = string("transpose_4")];
+            tensor<fp16, [1024, 32, 128]> var_4673_cast_fp16 = pow(x = x_169_cast_fp16, y = var_3985_promoted_1_to_fp16)[name = string("op_4673_cast_fp16")];
+            tensor<int32, [1]> var_4675_axes_0 = const()[name = string("op_4675_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_4675_keep_dims_0 = const()[name = string("op_4675_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1024, 32, 1]> var_4675_cast_fp16 = reduce_mean(axes = var_4675_axes_0, keep_dims = var_4675_keep_dims_0, x = var_4673_cast_fp16)[name = string("op_4675_cast_fp16")];
+            fp16 var_4676_to_fp16 = const()[name = string("op_4676_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1024, 32, 1]> var_4677_cast_fp16 = add(x = var_4675_cast_fp16, y = var_4676_to_fp16)[name = string("op_4677_cast_fp16")];
+            fp32 var_4678_epsilon_0 = const()[name = string("op_4678_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1024, 32, 1]> var_4678_cast_fp16 = rsqrt(epsilon = var_4678_epsilon_0, x = var_4677_cast_fp16)[name = string("op_4678_cast_fp16")];
+            tensor<fp16, [1024, 32, 128]> x_171_cast_fp16 = mul(x = x_169_cast_fp16, y = var_4678_cast_fp16)[name = string("x_171_cast_fp16")];
+            tensor<fp16, [128]> layers_6_linear_attn_norm_weight_to_fp16 = const()[name = string("layers_6_linear_attn_norm_weight_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(688022016)))];
+            tensor<fp16, [1024, 32, 128]> var_4680_cast_fp16 = mul(x = layers_6_linear_attn_norm_weight_to_fp16, y = x_171_cast_fp16)[name = string("op_4680_cast_fp16")];
+            tensor<fp16, [1024, 32, 128]> var_4681_cast_fp16 = silu(x = input_95_cast_fp16)[name = string("op_4681_cast_fp16")];
+            tensor<fp16, [1024, 32, 128]> core_cast_fp16 = mul(x = var_4680_cast_fp16, y = var_4681_cast_fp16)[name = string("core_cast_fp16")];
+            tensor<int32, [3]> var_4683 = const()[name = string("op_4683"), val = tensor<int32, [3]>([1, 1024, 4096])];
+            tensor<fp16, [1, 1024, 4096]> input_97_cast_fp16 = reshape(shape = var_4683, x = core_cast_fp16)[name = string("input_97_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> linear_51_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers_6_linear_attn_out_proj_weight_cast_fp16, x = input_97_cast_fp16)[name = string("linear_51_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> x_173_cast_fp16 = add(x = x_151_cast_fp16, y = linear_51_cast_fp16)[name = string("x_173_cast_fp16")];
+            fp16 var_3985_promoted_2_to_fp16 = const()[name = string("op_3985_promoted_2_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 1024, 2560]> var_4689_cast_fp16 = pow(x = x_173_cast_fp16, y = var_3985_promoted_2_to_fp16)[name = string("op_4689_cast_fp16")];
+            tensor<int32, [1]> var_4691_axes_0 = const()[name = string("op_4691_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_4691_keep_dims_0 = const()[name = string("op_4691_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 1024, 1]> var_4691_cast_fp16 = reduce_mean(axes = var_4691_axes_0, keep_dims = var_4691_keep_dims_0, x = var_4689_cast_fp16)[name = string("op_4691_cast_fp16")];
+            fp16 var_4692_to_fp16 = const()[name = string("op_4692_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 1024, 1]> var_4693_cast_fp16 = add(x = var_4691_cast_fp16, y = var_4692_to_fp16)[name = string("op_4693_cast_fp16")];
+            fp32 var_4694_epsilon_0 = const()[name = string("op_4694_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 1024, 1]> var_4694_cast_fp16 = rsqrt(epsilon = var_4694_epsilon_0, x = var_4693_cast_fp16)[name = string("op_4694_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> x_175_cast_fp16 = mul(x = x_173_cast_fp16, y = var_4694_cast_fp16)[name = string("x_175_cast_fp16")];
+            tensor<fp16, [2560]> var_4697_to_fp16 = const()[name = string("op_4697_to_fp16"), val = tensor<fp16, [2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(688022336)))];
+            tensor<fp16, [1, 1024, 2560]> input_99_cast_fp16 = mul(x = x_175_cast_fp16, y = var_4697_to_fp16)[name = string("input_99_cast_fp16")];
+            tensor<fp16, [1, 1024, 9216]> linear_52_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = layers_6_mlp_gate_proj_weight_cast_fp16, x = input_99_cast_fp16)[name = string("linear_52_cast_fp16")];
+            tensor<fp16, [1, 1024, 9216]> var_4704_cast_fp16 = silu(x = linear_52_cast_fp16)[name = string("op_4704_cast_fp16")];
+            tensor<fp16, [1, 1024, 9216]> linear_53_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = layers_6_mlp_up_proj_weight_cast_fp16, x = input_99_cast_fp16)[name = string("linear_53_cast_fp16")];
+            tensor<fp16, [1, 1024, 9216]> input_103_cast_fp16 = mul(x = var_4704_cast_fp16, y = linear_53_cast_fp16)[name = string("input_103_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> linear_54_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers_6_mlp_down_proj_weight_cast_fp16, x = input_103_cast_fp16)[name = string("linear_54_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> x_177_cast_fp16 = add(x = x_173_cast_fp16, y = linear_54_cast_fp16)[name = string("x_177_cast_fp16")];
+            int32 var_4724 = const()[name = string("op_4724"), val = int32(-1)];
+            fp16 var_4723_promoted_to_fp16 = const()[name = string("op_4723_promoted_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 1024, 2560]> var_4733_cast_fp16 = pow(x = x_177_cast_fp16, y = var_4723_promoted_to_fp16)[name = string("op_4733_cast_fp16")];
+            tensor<int32, [1]> var_4735_axes_0 = const()[name = string("op_4735_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_4735_keep_dims_0 = const()[name = string("op_4735_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 1024, 1]> var_4735_cast_fp16 = reduce_mean(axes = var_4735_axes_0, keep_dims = var_4735_keep_dims_0, x = var_4733_cast_fp16)[name = string("op_4735_cast_fp16")];
+            fp16 var_4736_to_fp16 = const()[name = string("op_4736_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 1024, 1]> var_4737_cast_fp16 = add(x = var_4735_cast_fp16, y = var_4736_to_fp16)[name = string("op_4737_cast_fp16")];
+            fp32 var_4738_epsilon_0 = const()[name = string("op_4738_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 1024, 1]> var_4738_cast_fp16 = rsqrt(epsilon = var_4738_epsilon_0, x = var_4737_cast_fp16)[name = string("op_4738_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> x_179_cast_fp16 = mul(x = x_177_cast_fp16, y = var_4738_cast_fp16)[name = string("x_179_cast_fp16")];
+            tensor<fp16, [2560]> var_4741_to_fp16 = const()[name = string("op_4741_to_fp16"), val = tensor<fp16, [2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(688027520)))];
+            tensor<fp16, [1, 1024, 2560]> x_181_cast_fp16 = mul(x = x_179_cast_fp16, y = var_4741_to_fp16)[name = string("x_181_cast_fp16")];
+            tensor<fp16, [1, 1024, 8192]> linear_55_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = layers_7_self_attn_q_proj_weight_cast_fp16, x = x_181_cast_fp16)[name = string("linear_55_cast_fp16")];
+            tensor<int32, [4]> var_4752 = const()[name = string("op_4752"), val = tensor<int32, [4]>([1, 1024, 16, 512])];
+            tensor<fp16, [1, 1024, 16, 512]> qg_cast_fp16 = reshape(shape = var_4752, x = linear_55_cast_fp16)[name = string("qg_cast_fp16")];
+            tensor<int32, [4]> x_183_begin_0 = const()[name = string("x_183_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_183_end_0 = const()[name = string("x_183_end_0"), val = tensor<int32, [4]>([1, 1024, 16, 256])];
+            tensor<bool, [4]> x_183_end_mask_0 = const()[name = string("x_183_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1, 1024, 16, 256]> x_183_cast_fp16 = slice_by_index(begin = x_183_begin_0, end = x_183_end_0, end_mask = x_183_end_mask_0, x = qg_cast_fp16)[name = string("x_183_cast_fp16")];
+            tensor<int32, [4]> gate_5_begin_0 = const()[name = string("gate_5_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 256])];
+            tensor<int32, [4]> gate_5_end_0 = const()[name = string("gate_5_end_0"), val = tensor<int32, [4]>([1, 1024, 16, 512])];
+            tensor<bool, [4]> gate_5_end_mask_0 = const()[name = string("gate_5_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1, 1024, 16, 256]> gate_5_cast_fp16 = slice_by_index(begin = gate_5_begin_0, end = gate_5_end_0, end_mask = gate_5_end_mask_0, x = qg_cast_fp16)[name = string("gate_5_cast_fp16")];
+            tensor<int32, [3]> var_4756 = const()[name = string("op_4756"), val = tensor<int32, [3]>([1, 1024, 4096])];
+            tensor<fp16, [1, 1024, 4096]> gate_cast_fp16 = reshape(shape = var_4756, x = gate_5_cast_fp16)[name = string("gate_cast_fp16")];
+            fp16 var_4723_promoted_1_to_fp16 = const()[name = string("op_4723_promoted_1_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 1024, 16, 256]> var_4759_cast_fp16 = pow(x = x_183_cast_fp16, y = var_4723_promoted_1_to_fp16)[name = string("op_4759_cast_fp16")];
+            tensor<int32, [1]> var_4761_axes_0 = const()[name = string("op_4761_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_4761_keep_dims_0 = const()[name = string("op_4761_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 1024, 16, 1]> var_4761_cast_fp16 = reduce_mean(axes = var_4761_axes_0, keep_dims = var_4761_keep_dims_0, x = var_4759_cast_fp16)[name = string("op_4761_cast_fp16")];
+            fp16 var_4762_to_fp16 = const()[name = string("op_4762_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 1024, 16, 1]> var_4763_cast_fp16 = add(x = var_4761_cast_fp16, y = var_4762_to_fp16)[name = string("op_4763_cast_fp16")];
+            fp32 var_4764_epsilon_0 = const()[name = string("op_4764_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 1024, 16, 1]> var_4764_cast_fp16 = rsqrt(epsilon = var_4764_epsilon_0, x = var_4763_cast_fp16)[name = string("op_4764_cast_fp16")];
+            tensor<fp16, [1, 1024, 16, 256]> x_185_cast_fp16 = mul(x = x_183_cast_fp16, y = var_4764_cast_fp16)[name = string("x_185_cast_fp16")];
+            tensor<fp16, [256]> var_4767_to_fp16 = const()[name = string("op_4767_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(688032704)))];
+            tensor<fp16, [1, 1024, 16, 256]> var_4768_cast_fp16 = mul(x = x_185_cast_fp16, y = var_4767_to_fp16)[name = string("op_4768_cast_fp16")];
+            tensor<int32, [4]> q_65_perm_0 = const()[name = string("q_65_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
+            tensor<fp16, [1, 1024, 1024]> linear_56_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = layers_7_self_attn_k_proj_weight_cast_fp16, x = x_181_cast_fp16)[name = string("linear_56_cast_fp16")];
+            tensor<int32, [4]> var_4772 = const()[name = string("op_4772"), val = tensor<int32, [4]>([1, 1024, 4, 256])];
+            tensor<fp16, [1, 1024, 4, 256]> x_187_cast_fp16 = reshape(shape = var_4772, x = linear_56_cast_fp16)[name = string("x_187_cast_fp16")];
+            fp16 var_4723_promoted_2_to_fp16 = const()[name = string("op_4723_promoted_2_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 1024, 4, 256]> var_4775_cast_fp16 = pow(x = x_187_cast_fp16, y = var_4723_promoted_2_to_fp16)[name = string("op_4775_cast_fp16")];
+            tensor<int32, [1]> var_4777_axes_0 = const()[name = string("op_4777_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_4777_keep_dims_0 = const()[name = string("op_4777_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 1024, 4, 1]> var_4777_cast_fp16 = reduce_mean(axes = var_4777_axes_0, keep_dims = var_4777_keep_dims_0, x = var_4775_cast_fp16)[name = string("op_4777_cast_fp16")];
+            fp16 var_4778_to_fp16 = const()[name = string("op_4778_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 1024, 4, 1]> var_4779_cast_fp16 = add(x = var_4777_cast_fp16, y = var_4778_to_fp16)[name = string("op_4779_cast_fp16")];
+            fp32 var_4780_epsilon_0 = const()[name = string("op_4780_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 1024, 4, 1]> var_4780_cast_fp16 = rsqrt(epsilon = var_4780_epsilon_0, x = var_4779_cast_fp16)[name = string("op_4780_cast_fp16")];
+            tensor<fp16, [1, 1024, 4, 256]> x_189_cast_fp16 = mul(x = x_187_cast_fp16, y = var_4780_cast_fp16)[name = string("x_189_cast_fp16")];
+            tensor<fp16, [256]> var_4783_to_fp16 = const()[name = string("op_4783_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(688033280)))];
+            tensor<fp16, [1, 1024, 4, 256]> var_4784_cast_fp16 = mul(x = x_189_cast_fp16, y = var_4783_to_fp16)[name = string("op_4784_cast_fp16")];
+            tensor<int32, [4]> k_55_perm_0 = const()[name = string("k_55_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
+            tensor<fp16, [1, 1024, 1024]> linear_57_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = layers_7_self_attn_v_proj_weight_cast_fp16, x = x_181_cast_fp16)[name = string("linear_57_cast_fp16")];
+            tensor<int32, [4]> var_4788 = const()[name = string("op_4788"), val = tensor<int32, [4]>([1, 1024, 4, 256])];
+            tensor<fp16, [1, 1024, 4, 256]> var_4789_cast_fp16 = reshape(shape = var_4788, x = linear_57_cast_fp16)[name = string("op_4789_cast_fp16")];
+            tensor<int32, [4]> v_29_perm_0 = const()[name = string("v_29_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
+            tensor<int32, [4]> var_4795_begin_0 = const()[name = string("op_4795_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_4795_end_0 = const()[name = string("op_4795_end_0"), val = tensor<int32, [4]>([1, 16, 1024, 64])];
+            tensor<bool, [4]> var_4795_end_mask_0 = const()[name = string("op_4795_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1, 16, 1024, 256]> q_65_cast_fp16 = transpose(perm = q_65_perm_0, x = var_4768_cast_fp16)[name = string("transpose_3")];
+            tensor<fp16, [1, 16, 1024, 64]> var_4795_cast_fp16 = slice_by_index(begin = var_4795_begin_0, end = var_4795_end_0, end_mask = var_4795_end_mask_0, x = q_65_cast_fp16)[name = string("op_4795_cast_fp16")];
+            tensor<fp16, [1, 16, 1024, 64]> var_4796_cast_fp16 = mul(x = var_4795_cast_fp16, y = c_1_cast_fp16)[name = string("op_4796_cast_fp16")];
+            tensor<int32, [4]> var_4803_begin_0 = const()[name = string("op_4803_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 32])];
+            tensor<int32, [4]> var_4803_end_0 = const()[name = string("op_4803_end_0"), val = tensor<int32, [4]>([1, 16, 1024, 64])];
+            tensor<bool, [4]> var_4803_end_mask_0 = const()[name = string("op_4803_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1, 16, 1024, 32]> var_4803_cast_fp16 = slice_by_index(begin = var_4803_begin_0, end = var_4803_end_0, end_mask = var_4803_end_mask_0, x = var_4795_cast_fp16)[name = string("op_4803_cast_fp16")];
+            fp16 const_79_promoted_to_fp16 = const()[name = string("const_79_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1, 16, 1024, 32]> var_4804_cast_fp16 = mul(x = var_4803_cast_fp16, y = const_79_promoted_to_fp16)[name = string("op_4804_cast_fp16")];
+            tensor<int32, [4]> var_4805_begin_0 = const()[name = string("op_4805_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_4805_end_0 = const()[name = string("op_4805_end_0"), val = tensor<int32, [4]>([1, 16, 1024, 32])];
+            tensor<bool, [4]> var_4805_end_mask_0 = const()[name = string("op_4805_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1, 16, 1024, 32]> var_4805_cast_fp16 = slice_by_index(begin = var_4805_begin_0, end = var_4805_end_0, end_mask = var_4805_end_mask_0, x = var_4795_cast_fp16)[name = string("op_4805_cast_fp16")];
+            bool var_4807_interleave_0 = const()[name = string("op_4807_interleave_0"), val = bool(false)];
+            tensor<fp16, [1, 16, 1024, 64]> var_4807_cast_fp16 = concat(axis = var_4724, interleave = var_4807_interleave_0, values = (var_4804_cast_fp16, var_4805_cast_fp16))[name = string("op_4807_cast_fp16")];
+            tensor<fp16, [1, 16, 1024, 64]> var_4808_cast_fp16 = mul(x = var_4807_cast_fp16, y = s_1_cast_fp16)[name = string("op_4808_cast_fp16")];
+            tensor<fp16, [1, 16, 1024, 64]> var_4809_cast_fp16 = add(x = var_4796_cast_fp16, y = var_4808_cast_fp16)[name = string("op_4809_cast_fp16")];
+            tensor<int32, [4]> var_4810_begin_0 = const()[name = string("op_4810_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
+            tensor<int32, [4]> var_4810_end_0 = const()[name = string("op_4810_end_0"), val = tensor<int32, [4]>([1, 16, 1024, 256])];
+            tensor<bool, [4]> var_4810_end_mask_0 = const()[name = string("op_4810_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1, 16, 1024, 192]> var_4810_cast_fp16 = slice_by_index(begin = var_4810_begin_0, end = var_4810_end_0, end_mask = var_4810_end_mask_0, x = q_65_cast_fp16)[name = string("op_4810_cast_fp16")];
+            bool q_interleave_0 = const()[name = string("q_interleave_0"), val = bool(false)];
+            tensor<fp16, [1, 16, 1024, 256]> q_cast_fp16 = concat(axis = var_4724, interleave = q_interleave_0, values = (var_4809_cast_fp16, var_4810_cast_fp16))[name = string("q_cast_fp16")];
+            tensor<int32, [4]> var_4813_begin_0 = const()[name = string("op_4813_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_4813_end_0 = const()[name = string("op_4813_end_0"), val = tensor<int32, [4]>([1, 4, 1024, 64])];
+            tensor<bool, [4]> var_4813_end_mask_0 = const()[name = string("op_4813_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1, 4, 1024, 256]> k_55_cast_fp16 = transpose(perm = k_55_perm_0, x = var_4784_cast_fp16)[name = string("transpose_2")];
+            tensor<fp16, [1, 4, 1024, 64]> var_4813_cast_fp16 = slice_by_index(begin = var_4813_begin_0, end = var_4813_end_0, end_mask = var_4813_end_mask_0, x = k_55_cast_fp16)[name = string("op_4813_cast_fp16")];
+            tensor<fp16, [1, 4, 1024, 64]> var_4814_cast_fp16 = mul(x = var_4813_cast_fp16, y = c_1_cast_fp16)[name = string("op_4814_cast_fp16")];
+            tensor<int32, [4]> var_4821_begin_0 = const()[name = string("op_4821_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 32])];
+            tensor<int32, [4]> var_4821_end_0 = const()[name = string("op_4821_end_0"), val = tensor<int32, [4]>([1, 4, 1024, 64])];
+            tensor<bool, [4]> var_4821_end_mask_0 = const()[name = string("op_4821_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1, 4, 1024, 32]> var_4821_cast_fp16 = slice_by_index(begin = var_4821_begin_0, end = var_4821_end_0, end_mask = var_4821_end_mask_0, x = var_4813_cast_fp16)[name = string("op_4821_cast_fp16")];
+            fp16 const_81_promoted_to_fp16 = const()[name = string("const_81_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1, 4, 1024, 32]> var_4822_cast_fp16 = mul(x = var_4821_cast_fp16, y = const_81_promoted_to_fp16)[name = string("op_4822_cast_fp16")];
+            tensor<int32, [4]> var_4823_begin_0 = const()[name = string("op_4823_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_4823_end_0 = const()[name = string("op_4823_end_0"), val = tensor<int32, [4]>([1, 4, 1024, 32])];
+            tensor<bool, [4]> var_4823_end_mask_0 = const()[name = string("op_4823_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1, 4, 1024, 32]> var_4823_cast_fp16 = slice_by_index(begin = var_4823_begin_0, end = var_4823_end_0, end_mask = var_4823_end_mask_0, x = var_4813_cast_fp16)[name = string("op_4823_cast_fp16")];
+            bool var_4825_interleave_0 = const()[name = string("op_4825_interleave_0"), val = bool(false)];
+            tensor<fp16, [1, 4, 1024, 64]> var_4825_cast_fp16 = concat(axis = var_4724, interleave = var_4825_interleave_0, values = (var_4822_cast_fp16, var_4823_cast_fp16))[name = string("op_4825_cast_fp16")];
+            tensor<fp16, [1, 4, 1024, 64]> var_4826_cast_fp16 = mul(x = var_4825_cast_fp16, y = s_1_cast_fp16)[name = string("op_4826_cast_fp16")];
+            tensor<fp16, [1, 4, 1024, 64]> var_4827_cast_fp16 = add(x = var_4814_cast_fp16, y = var_4826_cast_fp16)[name = string("op_4827_cast_fp16")];
+            tensor<int32, [4]> var_4828_begin_0 = const()[name = string("op_4828_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
+            tensor<int32, [4]> var_4828_end_0 = const()[name = string("op_4828_end_0"), val = tensor<int32, [4]>([1, 4, 1024, 256])];
+            tensor<bool, [4]> var_4828_end_mask_0 = const()[name = string("op_4828_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1, 4, 1024, 192]> var_4828_cast_fp16 = slice_by_index(begin = var_4828_begin_0, end = var_4828_end_0, end_mask = var_4828_end_mask_0, x = k_55_cast_fp16)[name = string("op_4828_cast_fp16")];
+            bool k_57_interleave_0 = const()[name = string("k_57_interleave_0"), val = bool(false)];
+            tensor<fp16, [1, 4, 1024, 256]> k_57_cast_fp16 = concat(axis = var_4724, interleave = k_57_interleave_0, values = (var_4827_cast_fp16, var_4828_cast_fp16))[name = string("k_57_cast_fp16")];
+            tensor<int32, [1]> var_4833_axes_0 = const()[name = string("op_4833_axes_0"), val = tensor<int32, [1]>([2])];
+            tensor<fp16, [1, 4, 1, 1024, 256]> var_4833_cast_fp16 = expand_dims(axes = var_4833_axes_0, x = k_57_cast_fp16)[name = string("op_4833_cast_fp16")];
+            tensor<int32, [5]> var_4835_reps_0 = const()[name = string("op_4835_reps_0"), val = tensor<int32, [5]>([1, 1, 4, 1, 1])];
+            tensor<fp16, [1, 4, 4, 1024, 256]> var_4835_cast_fp16 = tile(reps = var_4835_reps_0, x = var_4833_cast_fp16)[name = string("op_4835_cast_fp16")];
+            tensor<int32, [4]> var_4836 = const()[name = string("op_4836"), val = tensor<int32, [4]>([1, 16, 1024, 256])];
+            tensor<fp16, [1, 16, 1024, 256]> k_cast_fp16 = reshape(shape = var_4836, x = var_4835_cast_fp16)[name = string("k_cast_fp16")];
+            tensor<int32, [1]> var_4840_axes_0 = const()[name = string("op_4840_axes_0"), val = tensor<int32, [1]>([2])];
+            tensor<fp16, [1, 4, 1024, 256]> v_29_cast_fp16 = transpose(perm = v_29_perm_0, x = var_4789_cast_fp16)[name = string("transpose_1")];
+            tensor<fp16, [1, 4, 1, 1024, 256]> var_4840_cast_fp16 = expand_dims(axes = var_4840_axes_0, x = v_29_cast_fp16)[name = string("op_4840_cast_fp16")];
+            tensor<int32, [5]> var_4842_reps_0 = const()[name = string("op_4842_reps_0"), val = tensor<int32, [5]>([1, 1, 4, 1, 1])];
+            tensor<fp16, [1, 4, 4, 1024, 256]> var_4842_cast_fp16 = tile(reps = var_4842_reps_0, x = var_4840_cast_fp16)[name = string("op_4842_cast_fp16")];
+            tensor<int32, [4]> var_4843 = const()[name = string("op_4843"), val = tensor<int32, [4]>([1, 16, 1024, 256])];
+            tensor<fp16, [1, 16, 1024, 256]> v_cast_fp16 = reshape(shape = var_4843, x = var_4842_cast_fp16)[name = string("v_cast_fp16")];
+            bool var_4846_transpose_x_1 = const()[name = string("op_4846_transpose_x_1"), val = bool(false)];
+            bool var_4846_transpose_y_1 = const()[name = string("op_4846_transpose_y_1"), val = bool(true)];
+            tensor<fp16, [1, 16, 1024, 1024]> var_4846_cast_fp16 = matmul(transpose_x = var_4846_transpose_x_1, transpose_y = var_4846_transpose_y_1, x = q_cast_fp16, y = k_cast_fp16)[name = string("op_4846_cast_fp16")];
+            fp16 var_4847_to_fp16 = const()[name = string("op_4847_to_fp16"), val = fp16(0x1p-4)];
+            tensor<fp16, [1, 16, 1024, 1024]> var_4848_cast_fp16 = mul(x = var_4846_cast_fp16, y = var_4847_to_fp16)[name = string("op_4848_cast_fp16")];
+            tensor<fp16, [1, 16, 1024, 1024]> scores_cast_fp16 = add(x = var_4848_cast_fp16, y = mask_to_fp16)[name = string("scores_cast_fp16")];
+            tensor<fp16, [1, 16, 1024, 1024]> var_4850_cast_fp16 = softmax(axis = var_4724, x = scores_cast_fp16)[name = string("op_4850_cast_fp16")];
+            bool out_5_transpose_x_0 = const()[name = string("out_5_transpose_x_0"), val = bool(false)];
+            bool out_5_transpose_y_0 = const()[name = string("out_5_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1, 16, 1024, 256]> out_5_cast_fp16 = matmul(transpose_x = out_5_transpose_x_0, transpose_y = out_5_transpose_y_0, x = var_4850_cast_fp16, y = v_cast_fp16)[name = string("out_5_cast_fp16")];
+            tensor<int32, [4]> var_4852_perm_0 = const()[name = string("op_4852_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
+            tensor<int32, [3]> var_4853 = const()[name = string("op_4853"), val = tensor<int32, [3]>([1, 1024, 4096])];
+            tensor<fp16, [1, 1024, 16, 256]> var_4852_cast_fp16 = transpose(perm = var_4852_perm_0, x = out_5_cast_fp16)[name = string("transpose_0")];
+            tensor<fp16, [1, 1024, 4096]> out_cast_fp16 = reshape(shape = var_4853, x = var_4852_cast_fp16)[name = string("out_cast_fp16")];
+            tensor<fp16, [1, 1024, 4096]> var_4855_cast_fp16 = sigmoid(x = gate_cast_fp16)[name = string("op_4855_cast_fp16")];
+            tensor<fp16, [1, 1024, 4096]> input_105_cast_fp16 = mul(x = out_cast_fp16, y = var_4855_cast_fp16)[name = string("input_105_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> linear_58_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers_7_self_attn_o_proj_weight_cast_fp16, x = input_105_cast_fp16)[name = string("linear_58_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> x_195_cast_fp16 = add(x = x_177_cast_fp16, y = linear_58_cast_fp16)[name = string("x_195_cast_fp16")];
+            fp16 var_4723_promoted_3_to_fp16 = const()[name = string("op_4723_promoted_3_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 1024, 2560]> var_4861_cast_fp16 = pow(x = x_195_cast_fp16, y = var_4723_promoted_3_to_fp16)[name = string("op_4861_cast_fp16")];
+            tensor<int32, [1]> var_4863_axes_0 = const()[name = string("op_4863_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_4863_keep_dims_0 = const()[name = string("op_4863_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 1024, 1]> var_4863_cast_fp16 = reduce_mean(axes = var_4863_axes_0, keep_dims = var_4863_keep_dims_0, x = var_4861_cast_fp16)[name = string("op_4863_cast_fp16")];
+            fp16 var_4864_to_fp16 = const()[name = string("op_4864_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 1024, 1]> var_4865_cast_fp16 = add(x = var_4863_cast_fp16, y = var_4864_to_fp16)[name = string("op_4865_cast_fp16")];
+            fp32 var_4866_epsilon_0 = const()[name = string("op_4866_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 1024, 1]> var_4866_cast_fp16 = rsqrt(epsilon = var_4866_epsilon_0, x = var_4865_cast_fp16)[name = string("op_4866_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> x_cast_fp16 = mul(x = x_195_cast_fp16, y = var_4866_cast_fp16)[name = string("x_cast_fp16")];
+            tensor<fp16, [2560]> var_4869_to_fp16 = const()[name = string("op_4869_to_fp16"), val = tensor<fp16, [2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(688033856)))];
+            tensor<fp16, [1, 1024, 2560]> input_107_cast_fp16 = mul(x = x_cast_fp16, y = var_4869_to_fp16)[name = string("input_107_cast_fp16")];
+            tensor<fp16, [1, 1024, 9216]> linear_59_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = layers_7_mlp_gate_proj_weight_cast_fp16, x = input_107_cast_fp16)[name = string("linear_59_cast_fp16")];
+            tensor<fp16, [1, 1024, 9216]> var_4876_cast_fp16 = silu(x = linear_59_cast_fp16)[name = string("op_4876_cast_fp16")];
+            tensor<fp16, [1, 1024, 9216]> linear_60_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = layers_7_mlp_up_proj_weight_cast_fp16, x = input_107_cast_fp16)[name = string("linear_60_cast_fp16")];
+            tensor<fp16, [1, 1024, 9216]> input_cast_fp16 = mul(x = var_4876_cast_fp16, y = linear_60_cast_fp16)[name = string("input_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> linear_61_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers_7_mlp_down_proj_weight_cast_fp16, x = input_cast_fp16)[name = string("linear_61_cast_fp16")];
+            tensor<fp16, [1, 1024, 2560]> hidden_out = add(x = x_195_cast_fp16, y = linear_61_cast_fp16)[name = string("op_4882_cast_fp16")];
+        } -> (hidden_out);
+}
\ No newline at end of file